Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.21433504 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.04458782 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.044508774 Region set with 3 regions was created /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1185 1186 1187 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.492604885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.493599367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.493629083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189} {"msg":"Crucible c0b24c18-8748-42ce-be80-37992a412940 has session id: c146b8a6-d13e-40c3-a2c6-bf4c3550d6da","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.493800807Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Upstairs opts: Upstairs UUID: c0b24c18-8748-42ce-be80-37992a412940, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.493910231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Crucible stats registered with UUID: c0b24c18-8748-42ce-be80-37992a412940","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.493946875Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.49492848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189} {"msg":"c0b24c18-8748-42ce-be80-37992a412940 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495028635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495060301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495086597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495109603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.49513395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495158406Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495188122Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495210848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495234525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495296246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.49559951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495636825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495664291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495690407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495735141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495763636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495790292Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495810629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495832516Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495861362Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495887718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495909395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495931021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:41.495957508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495978165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.495999871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.496023548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:41.496045844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.496574846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_connection connected from Ok(127.0.0.1:39098)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.496829799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.497250866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.497332174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_connection connected from Ok(127.0.0.1:42929)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.4973578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_connection connected from Ok(127.0.0.1:56741)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.497574228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"io task","client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.498765221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.498799256Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.498826842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.498854928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.498883454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.498901441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.499604367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.499736588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.499781331Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500202679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500238853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500265649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500368354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500387841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500401649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500431625Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500445732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500563855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500581842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.50059637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500609898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500623416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500637354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500650722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.50066407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500678078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500691146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500704914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500720822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.50073376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"":"downstairs","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500749327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500761296Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"0","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500774414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"1","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500787352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"client":"2","":"downstairs","session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.5008005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500812778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"c0b24c18-8748-42ce-be80-37992a412940 is now active with session: c146b8a6-d13e-40c3-a2c6-bf4c3550d6da","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500825676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500837544Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189,"session_id":"c146b8a6-d13e-40c3-a2c6-bf4c3550d6da"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:51.500849552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 6 Acked 1005 Read 256 Done Done Done false 7 Acked 1006 Read 88 Done Done Done false 8 Acked 1007 Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 8 7 6 5 2 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 9 8 7 6 5 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:55.589562139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1189} Begin loop 1 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.617673742Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.61809464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618129925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191} {"msg":"Crucible 04d46d30-5c64-4490-926a-55aaaf88eb31 has session id: 2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618315297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Upstairs opts: Upstairs UUID: 04d46d30-5c64-4490-926a-55aaaf88eb31, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.61842754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Crucible stats registered with UUID: 04d46d30-5c64-4490-926a-55aaaf88eb31","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618454807Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618736855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191} {"msg":"04d46d30-5c64-4490-926a-55aaaf88eb31 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618848948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618867155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618882133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618895941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618909409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618931276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618953882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.61897435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.618992747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.619007505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_connection connected from Ok(127.0.0.1:63055)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.619349074Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_connection connected from Ok(127.0.0.1:46116)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.619391788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:55.619423643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:55.619452319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:08:55.619476345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.619489673Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.619502671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.619624493Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.620004987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.620043661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.620074287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.620101443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.620695284Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.620731009Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.621254221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.621281148Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.621306714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.621319682Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.62133353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:08:55.621346868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.621045703Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_connection connected from Ok(127.0.0.1:50134)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.621250712Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"io task","client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.622180234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.62220809Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.622829508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623430409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623458225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623474932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.62349326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623508127Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623521925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623535454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623549451Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623563139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623576637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623590725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Max found gen is 2","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-08-05T19:09:05.623607343Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623620871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Next flush: 3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623633869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623658265Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"":"downstairs","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623672203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623684012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"0","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.62369707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"1","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623712567Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"client":"2","":"downstairs","session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623726115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623738443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"04d46d30-5c64-4490-926a-55aaaf88eb31 is now active with session: 2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623751142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.62376291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191,"session_id":"2b8b4b8c-844f-4d3a-a4a8-8ac8f4986652"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:05.623775198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 4096 data: 2 02/30 Read block 463 len 16384 03/30 Read block 521 len 4096 04/30 Read block 92 len 4096 05/30 Read block 327 len 20480 06/30 Read block 217 len 40960 07/30 Read block 371 len 20480 08/30 Read block 425 len 4096 09/30 Write block 558 len 32768 data: 2 2 2 2 2 2 2 2 10/30 Write block 514 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Write block 485 len 32768 data: 2 2 2 2 2 2 2 2 12/30 Write block 320 len 28672 data: 2 2 2 2 2 2 2 13/30 Read block 363 len 32768 14/30 Write block 406 len 8192 data: 2 2 15/30 Read block 526 len 28672 16/30 Write block 480 len 24576 data: 2 2 2 2 2 3 17/30 Read block 415 len 8192 18/30 Read block 242 len 20480 19/30 Flush 20/30 Write block 458 len 20480 data: 2 2 2 2 2 21/30 Read block 300 len 24576 22/30 Read block 148 len 8192 23/30 Write block 114 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Read block 116 len 28672 25/30 Read block 80 len 4096 26/30 Write block 229 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Write block 479 len 4096 data: 2 28/30 Write block 250 len 36864 data: 2 2 2 2 2 2 2 2 2 29/30 Write block 396 len 12288 data: 2 2 2 30/30 Write block 113 len 16384 data: 2 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 1 Done Done Done false 2 Acked 1001 Read 4 Done Done Done false 3 Acked 1002 Read 1 Done Done Done false 4 Acked 1003 Read 1 Done Done Done false 5 Acked 1004 Read 5 Done Done Done false 6 Acked 1005 Read 10 Done Done Done false 7 Acked 1006 Read 5 Done Done Done false 8 Acked 1007 Read 1 Done Done Done false 9 Acked 1008 Write 8 Done Done Done false 10 Acked 1009 Write 8 Done Done Done false 11 Acked 1010 Write 8 Done Done Done false 12 Acked 1011 Write 7 Done Done Done false 13 Acked 1012 Read 8 Done Done Done false 14 Acked 1013 Write 2 Done Done Done false 15 Acked 1014 Read 7 Done Done Done false 16 Acked 1015 Write 6 Done Done Done false 17 Acked 1016 Read 2 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Flush 0 Done Done Sent false 20 Acked 1019 Write 5 Done Done Sent false 21 Acked 1020 Read 6 Done Done Sent false 22 Acked 1021 Read 2 Done Done Sent false 23 Acked 1022 Write 10 Done Done Sent false 24 Acked 1023 Read 7 Done Done Sent false 25 Acked 1024 Read 1 Done Done Sent false 26 Acked 1025 Write 8 Done Done Sent false 27 Acked 1026 Write 1 Done Done Sent false 28 Acked 1027 Write 9 Done Done Sent false 29 Acked 1028 Write 3 Done Done Sent false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 12 14 Done 29 29 18 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 0 Downstairs last five completed: Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:05.646829151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1191} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 1 1 1 2 2 2 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 1 1 1 2 2 2 F F F 5 100-119 1 1 1 2 2 2 T T F <--- 6 120-139 1 1 1 2 2 2 T T F <--- 7 140-159 1 1 1 2 2 2 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 1 1 1 2 2 2 F F F 10 200-219 1 1 1 2 2 2 F F F 11 220-239 1 1 1 2 2 2 T T F <--- 12 240-259 1 1 1 2 2 2 T T F <--- 13 260-279 1 1 1 2 2 2 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 2 2 2 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 2 2 2 F F F 18 360-379 1 1 1 2 2 2 F F F 19 380-399 1 1 1 2 2 2 T T F <--- 20 400-419 2 2 2 3 3 3 F F F 21 420-439 1 1 1 2 2 2 F F F 22 440-459 1 1 1 2 2 2 T T F <--- 23 460-479 1 1 1 2 2 2 T T F <--- 24 480-499 2 2 2 3 3 3 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 3 3 3 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 1 1 1 2 2 2 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 1 1 1 2 2 2 F F F 4 080-099 1 1 1 2 2 2 F F F 5 100-119 3 3 3 4 4 4 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 1 1 1 2 2 2 F F F 8 160-179 1 1 1 2 2 2 F F F 9 180-199 1 1 1 2 2 2 F F F 10 200-219 1 1 1 2 2 2 F F F 11 220-239 3 3 3 4 4 4 F F F 12 240-259 3 3 3 4 4 4 F F F 13 260-279 1 1 1 2 2 2 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 2 2 2 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 2 2 2 F F F 18 360-379 1 1 1 2 2 2 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 1 1 1 2 2 2 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 2 2 2 3 3 3 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 3 3 3 F F F 28 560-579 2 2 2 3 3 3 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.98622677Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.986705939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.986732845Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199} {"msg":"Crucible ab665317-7d55-47f1-bee7-0f1961662b97 has session id: cebf1685-bebf-45bf-a15d-18127e2c8100","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.98690117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Upstairs opts: Upstairs UUID: ab665317-7d55-47f1-bee7-0f1961662b97, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.986944324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Crucible stats registered with UUID: ab665317-7d55-47f1-bee7-0f1961662b97","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.986965601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987257077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199} {"msg":"ab665317-7d55-47f1-bee7-0f1961662b97 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987415134Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987432971Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987452858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987482034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.98750778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987530757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987555313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.98757802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987594038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987607625Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_connection connected from Ok(127.0.0.1:49356)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987806716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_connection connected from Ok(127.0.0.1:53224)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.987947465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:20.987975631Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:20.987992968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:20.988024504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.988037432Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.98805066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.988158404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.98871996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.988746297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.988761564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.988775152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.989426026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.989453882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.990026047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.990064961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.990093887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.990110784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.990125002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:20.99013826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.989119571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_connection connected from Ok(127.0.0.1:58827)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.989235594Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"io task","client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.990306705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.990332231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.990983215Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991598794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991633648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 4, 4, 2, 2, 2, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991658335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991682491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991708357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 4, 4, 2, 2, 2, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991730944Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991753211Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991778477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 2, 2, 2, 4, 4, 2, 2, 2, 2, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991804423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 3, 3, 1, 1, 1, 1, 3]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991829099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991857195Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991884061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991910318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991940763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991967489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"":"downstairs","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.991993475Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992016842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"0","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992043028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"1","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992069884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"client":"2","":"downstairs","session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.99209635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992121686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"ab665317-7d55-47f1-bee7-0f1961662b97 is now active with session: cebf1685-bebf-45bf-a15d-18127e2c8100","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992147722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992178438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199,"session_id":"cebf1685-bebf-45bf-a15d-18127e2c8100"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:30.992203594Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 100 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Read block 569 len 40960 03/30 Read block 187 len 32768 04/30 Read block 7 len 20480 05/30 Read block 272 len 12288 06/30 Write block 368 len 32768 data: 2 2 2 2 2 2 2 2 07/30 Read block 486 len 12288 08/30 Read block 258 len 4096 09/30 Read block 23 len 36864 10/30 Write block 398 len 12288 data: 3 2 2 11/30 Read block 559 len 36864 12/30 Write block 378 len 28672 data: 2 2 2 2 2 2 2 13/30 Read block 288 len 40960 14/30 Read block 353 len 32768 15/30 Read block 151 len 4096 16/30 Flush 17/30 Write block 249 len 32768 data: 2 3 3 3 3 3 3 3 18/30 Read block 578 len 12288 19/30 Write block 219 len 12288 data: 2 2 2 20/30 Read block 561 len 36864 21/30 Write block 78 len 16384 data: 2 2 2 2 22/30 Write block 571 len 4096 data: 2 23/30 Write block 484 len 28672 data: 3 4 3 3 3 3 3 24/30 Write block 158 len 28672 data: 2 2 2 2 2 2 2 25/30 Flush 26/30 Write block 111 len 8192 data: 2 2 27/30 Read block 303 len 8192 28/30 Write block 486 len 24576 data: 4 4 4 4 4 3 29/30 Read block 476 len 8192 30/30 Write block 345 len 8192 data: 2 2 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 8 Done Done Done false 18 Acked 1017 Read 3 Done Done Done false 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Write 7 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Flush 0 Done Sent Done false 26 Acked 1025 Write 2 Done Sent Done false 27 Acked 1026 Read 2 Done Sent Done false 28 Acked 1027 Write 6 Done Sent Done false 29 Acked 1028 Read 2 Done Sent Done false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 6 1 8 Done 13 8 13 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1015 1024 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:31.100260127Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1199} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 1 1 1 2 2 2 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 4 4 4 6 6 6 F F F 4 080-099 4 4 4 6 6 6 F F F 5 100-119 4 4 4 5 5 5 T T T 6 120-139 3 3 3 4 4 4 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 1 1 1 2 2 2 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 4 4 4 6 6 6 F F F 13 260-279 1 1 1 2 2 2 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 2 2 2 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 2 2 2 T F T <--- 18 360-379 4 4 4 5 5 5 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 4 4 4 5 5 5 F F F 21 420-439 1 1 1 2 2 2 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 4 4 4 6 6 6 T T T 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 6 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 1 1 1 2 2 2 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 4 4 4 6 6 6 F F F 4 080-099 4 4 4 6 6 6 F F F 5 100-119 5 5 5 7 7 7 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 4 4 4 6 6 6 F F F 8 160-179 4 4 4 6 6 6 F F F 9 180-199 1 1 1 2 2 2 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 4 4 4 6 6 6 F F F 13 260-279 1 1 1 2 2 2 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 2 2 2 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 5 5 5 7 7 7 F F F 18 360-379 4 4 4 5 5 5 F F F 19 380-399 4 4 4 5 5 5 F F F 20 400-419 4 4 4 5 5 5 F F F 21 420-439 1 1 1 2 2 2 F F F 22 440-459 3 3 3 4 4 4 F F F 23 460-479 3 3 3 4 4 4 F F F 24 480-499 5 5 5 7 7 7 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 3 3 3 F F F 28 560-579 4 4 4 6 6 6 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 7 Begin loop 3 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.601038325Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.601562637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.601594473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207} {"msg":"Crucible 18ce8c03-3464-406f-922e-bb37593aaae9 has session id: 7b622750-ef1d-47bb-9804-98a1582fc98c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.601726203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Upstairs opts: Upstairs UUID: 18ce8c03-3464-406f-922e-bb37593aaae9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.601767007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Crucible stats registered with UUID: 18ce8c03-3464-406f-922e-bb37593aaae9","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.601783275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602149501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207} {"msg":"18ce8c03-3464-406f-922e-bb37593aaae9 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602179966Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602202283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602230878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602255205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602278711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602301938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602325664Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602347631Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602369988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602392305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_connection connected from Ok(127.0.0.1:54036)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602497659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52598)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602583176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:45.602733714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:45.602769869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:45.602789056Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602802394Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602815782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.602941573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.603274414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.60330106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.603316808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.603332515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.60397674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.604016684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.604556244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.60458301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.604598727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.604615355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.604628883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:45.604642071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.604794688Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33674)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.60491272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"io task","client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.60572044Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.605768353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606267819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606784043Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.60680467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[0]R flush_numbers[0..12]: [2, 2, 2, 6, 6, 7, 4, 6, 6, 2, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606819608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 4, 4, 5, 3, 4, 4, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606833875Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606848143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[1]R flush_numbers[0..12]: [2, 2, 2, 6, 6, 7, 4, 6, 6, 2, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606861621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 4, 4, 5, 3, 4, 4, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606874969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606888997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[2]R flush_numbers[0..12]: [2, 2, 2, 6, 6, 7, 4, 6, 6, 2, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606902425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 4, 4, 5, 3, 4, 4, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606915653Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606929641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606942539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606959557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Next flush: 8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606978454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.606991902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"":"downstairs","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.60700504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607017068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"0","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607030336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"1","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607043504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"client":"2","":"downstairs","session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607056732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607068871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"18ce8c03-3464-406f-922e-bb37593aaae9 is now active with session: 7b622750-ef1d-47bb-9804-98a1582fc98c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607081659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607093547Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207,"session_id":"7b622750-ef1d-47bb-9804-98a1582fc98c"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:09:55.607105705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 448 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 190 len 36864 03/30 Read block 127 len 32768 04/30 Read block 228 len 20480 05/30 Write block 348 len 8192 data: 2 2 06/30 Write block 427 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 150 len 32768 08/30 Write block 24 len 24576 data: 2 2 2 2 2 2 09/30 Write block 467 len 4096 data: 2 10/30 Read block 531 len 40960 11/30 Flush 12/30 Write block 388 len 36864 data: 2 2 2 2 2 2 2 2 3 13/30 Flush 14/30 Write block 571 len 8192 data: 3 2 15/30 Write block 242 len 4096 data: 2 16/30 Read block 472 len 32768 17/30 Read block 79 len 20480 18/30 Read block 302 len 28672 19/30 Read block 146 len 36864 20/30 Write block 594 len 12288 data: 2 2 2 21/30 Read block 373 len 8192 22/30 Write block 310 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Write block 70 len 4096 data: 2 24/30 Write block 157 len 36864 data: 2 3 3 3 3 3 3 3 2 25/30 Write block 458 len 40960 data: 3 3 3 3 3 2 2 2 2 3 26/30 Read block 56 len 24576 27/30 Write block 108 len 32768 data: 3 2 2 3 3 3 4 4 28/30 Write block 109 len 40960 data: 3 3 4 4 4 5 5 4 3 3 29/30 Read block 82 len 4096 30/30 Write block 255 len 32768 data: 4 4 3 3 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 14 Acked 1013 Write 2 Done Done Done false 15 Acked 1014 Write 1 Done Done Done false 16 Acked 1015 Read 8 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Read 7 Done Done Done false 19 Acked 1018 Read 9 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:09:55.654006343Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1207} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 4 4 4 6 6 6 T T T 4 080-099 4 4 4 6 6 6 F F F 5 100-119 5 5 5 7 7 7 T T T 6 120-139 3 3 3 4 4 4 F F F 7 140-159 4 4 4 6 6 6 T T T 8 160-179 4 4 4 6 6 6 T T T 9 180-199 1 1 1 2 2 2 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 4 4 4 6 6 6 T T T 13 260-279 1 1 1 2 2 2 T T T 14 280-299 2 2 2 3 3 3 F F F 15 300-319 1 1 1 2 2 2 T T T 16 320-339 2 2 2 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 4 4 4 5 5 5 F F F 19 380-399 6 6 6 9 9 9 F F F 20 400-419 4 4 4 5 5 5 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 6 6 6 8 8 8 T T T 23 460-479 6 6 6 8 8 8 T T T 24 480-499 5 5 5 7 7 7 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 3 3 3 F F F 28 560-579 4 4 4 6 6 6 T T T 29 580-599 1 1 1 1 1 1 T T T Max gen: 6, Max flush: 9 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 7 7 7 10 10 10 F F F 4 080-099 4 4 4 6 6 6 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 7 7 7 10 10 10 F F F 9 180-199 1 1 1 2 2 2 F F F 10 200-219 4 4 4 6 6 6 F F F 11 220-239 4 4 4 6 6 6 F F F 12 240-259 7 7 7 10 10 10 F F F 13 260-279 7 7 7 10 10 10 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 7 7 7 10 10 10 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 4 4 4 5 5 5 F F F 19 380-399 6 6 6 9 9 9 F F F 20 400-419 4 4 4 5 5 5 F F F 21 420-439 6 6 6 8 8 8 F F F 22 440-459 7 7 7 10 10 10 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 5 5 5 7 7 7 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 3 3 3 F F F 28 560-579 7 7 7 10 10 10 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 7, Max flush: 10 Begin loop 4 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.649115568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.649928047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.649959373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215} {"msg":"Crucible aa916b6a-14ac-47cf-866e-ba081348135b has session id: c182be65-bdba-40de-976b-019e62de8ce1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.649984279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Upstairs opts: Upstairs UUID: aa916b6a-14ac-47cf-866e-ba081348135b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650019904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Crucible stats registered with UUID: aa916b6a-14ac-47cf-866e-ba081348135b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.65004732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650288574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215} {"msg":"aa916b6a-14ac-47cf-866e-ba081348135b active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650323029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650347835Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650372881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650397868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650422344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.65044746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650469937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650493573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.65051863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650541636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:10.650673357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47729)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650713171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:10.650744206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:10.650825444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650850141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650878796Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54768)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650904273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.650953645Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.651488696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.651521441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.651549067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.651575473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652086497Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652200571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652556048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652580894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652594652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652714444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.652732921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:10.65274643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.652033145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_connection connected from Ok(127.0.0.1:48523)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.652180644Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"io task","client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.653158839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.653189584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.653785746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654348022Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654375818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[0]R flush_numbers[0..12]: [2, 8, 2, 10, 6, 10, 4, 10, 10, 2, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654391966Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[0]R generation[0..12]: [1, 6, 1, 7, 4, 7, 3, 7, 7, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654406514Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654420822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[1]R flush_numbers[0..12]: [2, 8, 2, 10, 6, 10, 4, 10, 10, 2, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.65443426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[1]R generation[0..12]: [1, 6, 1, 7, 4, 7, 3, 7, 7, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.65450165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654521047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[2]R flush_numbers[0..12]: [2, 8, 2, 10, 6, 10, 4, 10, 10, 2, 6, 6]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654535245Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[2]R generation[0..12]: [1, 6, 1, 7, 4, 7, 3, 7, 7, 1, 4, 4]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654548643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.65456846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654581948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654595306Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654607994Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654620792Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"":"downstairs","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654644489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654657216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"0","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-08-05T19:10:20.654674144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"1","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654687912Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"client":"2","":"downstairs","session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.65470129Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654713608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"aa916b6a-14ac-47cf-866e-ba081348135b is now active with session: c182be65-bdba-40de-976b-019e62de8ce1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654726406Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654738175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215,"session_id":"c182be65-bdba-40de-976b-019e62de8ce1"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:20.654750243Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 249 len 20480 data: 3 4 4 4 4 02/30 Write block 199 len 28672 data: 2 2 2 2 2 2 2 03/30 Read block 546 len 24576 04/30 Flush 05/30 Flush 06/30 Write block 411 len 40960 data: 2 2 2 2 2 2 2 2 2 2 07/30 Write block 546 len 16384 data: 2 2 2 2 08/30 Read block 292 len 28672 09/30 Write block 78 len 28672 data: 3 3 3 3 2 2 2 10/30 Read block 93 len 16384 11/30 Write block 442 len 16384 data: 2 2 2 2 12/30 Flush 13/30 Write block 271 len 20480 data: 2 2 2 2 2 14/30 Read block 253 len 4096 15/30 Read block 427 len 12288 16/30 Read block 470 len 28672 17/30 Read block 375 len 40960 18/30 Read block 398 len 4096 19/30 Write block 504 len 36864 data: 2 2 2 2 2 2 2 2 2 20/30 Read block 269 len 16384 21/30 Read block 469 len 16384 22/30 Write block 195 len 36864 data: 2 2 2 2 3 3 3 3 3 23/30 Write block 400 len 36864 data: 3 2 2 2 2 2 3 3 2 24/30 Write block 373 len 32768 data: 3 3 3 2 2 3 3 3 25/30 Write block 213 len 36864 data: 2 2 2 2 2 2 3 3 3 26/30 Write block 497 len 8192 data: 2 2 27/30 Read block 28 len 24576 28/30 Read block 299 len 12288 29/30 Read block 166 len 40960 30/30 Write block 442 len 36864 data: 3 3 3 3 2 2 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 13 Acked 1012 Write 5 Done Done Done false 14 Acked 1013 Read 1 Done Done Done false 15 Acked 1014 Read 3 Done Done Done false 16 Acked 1015 Read 7 Done Done Done false 17 Acked 1016 Read 10 Done Done Done false 18 Acked 1017 Flush 0 Done Done Sent false 19 Acked 1018 Read 1 Done Done Sent false 20 Acked 1019 Write 9 Done Done Sent false 21 Acked 1020 Read 4 Done Done Sent false 22 Acked 1021 Read 4 Done Done Sent false 23 Acked 1022 Write 9 Done Done Sent false 24 Acked 1023 Write 9 Done Done Sent false 25 Acked 1024 Write 8 Done Done Sent false 26 Acked 1025 Write 9 Done Done Sent false 27 Acked 1026 Write 2 Done Done Sent false 28 Acked 1027 Read 6 Done Done Sent false 29 Acked 1028 Read 3 Done Done Sent false 30 Acked 1029 Read 10 Done Done Sent false 31 Acked 1030 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 14 16 Done 18 18 5 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:20.6814808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1215} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 7 7 7 10 10 10 F F F 9 180-199 8 8 8 11 11 11 T T F <--- 10 200-219 8 8 8 11 11 11 T T F <--- 11 220-239 4 4 4 6 6 6 T T F <--- 12 240-259 8 8 8 11 11 11 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 7 7 7 10 10 10 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 4 4 4 5 5 5 T T F <--- 19 380-399 6 6 6 9 9 9 T T F <--- 20 400-419 8 8 8 13 13 13 T T F <--- 21 420-439 8 8 8 13 13 13 F F F 22 440-459 8 8 8 13 13 13 T T F <--- 23 460-479 7 7 7 10 10 10 F F F 24 480-499 5 5 5 7 7 7 T T F <--- 25 500-519 2 2 2 3 3 3 T T T 26 520-539 2 2 2 3 3 3 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 7 7 7 10 10 10 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 8, Max flush: 14 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 7 7 7 10 10 10 F F F 8 160-179 7 7 7 10 10 10 F F F 9 180-199 9 9 9 15 15 15 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 8 8 8 11 11 11 F F F 13 260-279 8 8 8 14 14 14 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 7 7 7 10 10 10 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 6 6 6 8 8 8 F F F 18 360-379 9 9 9 15 15 15 F F F 19 380-399 9 9 9 15 15 15 F F F 20 400-419 9 9 9 15 15 15 F F F 21 420-439 8 8 8 13 13 13 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 15 15 15 F F F 25 500-519 9 9 9 15 15 15 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 8 8 8 13 13 13 F F F 28 560-579 7 7 7 10 10 10 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 9, Max flush: 15 Begin loop 5 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.507760161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508330886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508363861Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223} {"msg":"Crucible 59e965de-2019-41f4-ad09-11d7eb960fd4 has session id: 724423f9-787d-4c99-89be-bcd1147e1a72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508386638Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Upstairs opts: Upstairs UUID: 59e965de-2019-41f4-ad09-11d7eb960fd4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508419783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Crucible stats registered with UUID: 59e965de-2019-41f4-ad09-11d7eb960fd4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508445069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508738196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223} {"msg":"59e965de-2019-41f4-ad09-11d7eb960fd4 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508866927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508893103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.50890943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508923648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508937386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508952074Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.5089756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.508997177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509015125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509059988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_connection connected from Ok(127.0.0.1:64716)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509168512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:35.509197108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:35.509225494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:35.509258178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509280425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.50931419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_connection connected from Ok(127.0.0.1:52212)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509446521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509487715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.509997989Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.510027275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.510052551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.510079717Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.510659001Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.510696176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.511238905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.511266291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.511281619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.511293817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.511310774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:35.511324142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.510846683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_connection connected from Ok(127.0.0.1:61819)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.510939109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"io task","client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.511884779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.511918674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.51242066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.512968188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.512995994Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[0]R flush_numbers[0..12]: [2, 8, 2, 13, 13, 10, 4, 10, 10, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513012572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[0]R generation[0..12]: [1, 6, 1, 8, 8, 7, 3, 7, 7, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513041277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[1]R flush_numbers[0..12]: [2, 8, 2, 13, 13, 10, 4, 10, 10, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513054895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[1]R generation[0..12]: [1, 6, 1, 8, 8, 7, 3, 7, 7, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513068053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513086741Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[2]R flush_numbers[0..12]: [2, 8, 2, 13, 13, 10, 4, 10, 10, 15, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513100659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"[2]R generation[0..12]: [1, 6, 1, 8, 8, 7, 3, 7, 7, 9, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513113897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513131274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513144212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.51315719Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Next flush: 16","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513169969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513182676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"":"downstairs","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513195485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513207333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"0","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513220651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"1","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513233639Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"client":"2","":"downstairs","session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513246517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513258555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"59e965de-2019-41f4-ad09-11d7eb960fd4 is now active with session: 724423f9-787d-4c99-89be-bcd1147e1a72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513271254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513283122Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223,"session_id":"724423f9-787d-4c99-89be-bcd1147e1a72"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:10:45.513297339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 32768 data: 3 3 4 4 3 2 2 3 02/30 Flush 03/30 Write block 287 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Read block 553 len 16384 05/30 Read block 594 len 8192 06/30 Read block 528 len 16384 07/30 Write block 516 len 28672 data: 3 3 3 3 3 3 2 08/30 Read block 11 len 40960 09/30 Read block 277 len 40960 10/30 Write block 292 len 24576 data: 3 3 3 3 2 2 11/30 Write block 359 len 16384 data: 2 2 2 2 12/30 Read block 30 len 40960 13/30 Write block 266 len 8192 data: 2 2 14/30 Write block 554 len 12288 data: 2 2 2 15/30 Read block 291 len 36864 16/30 Flush 17/30 Write block 299 len 4096 data: 2 18/30 Write block 183 len 36864 data: 2 2 2 2 2 2 2 2 2 19/30 Write block 484 len 20480 data: 4 5 5 5 5 20/30 Read block 196 len 32768 21/30 Write block 173 len 32768 data: 2 2 2 2 2 2 2 2 22/30 Read block 371 len 4096 23/30 Write block 252 len 12288 data: 5 5 4 24/30 Write block 140 len 24576 data: 2 2 2 2 2 2 25/30 Write block 401 len 40960 data: 3 3 3 4 4 5 5 4 3 3 26/30 Read block 217 len 28672 27/30 Read block 44 len 8192 28/30 Write block 350 len 4096 data: 2 29/30 Read block 78 len 24576 30/30 Read block 575 len 16384 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 1 Done Done Done false 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:10:45.57873521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1223} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 7 7 7 10 10 10 T T T 8 160-179 7 7 7 10 10 10 T T T 9 180-199 9 9 9 15 15 15 T T T 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 8 8 8 11 11 11 T T T 13 260-279 10 10 10 17 17 17 F F F 14 280-299 10 10 10 17 17 17 T T T 15 300-319 7 7 7 10 10 10 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 10 10 10 17 17 17 T T T 18 360-379 10 10 10 17 17 17 F F F 19 380-399 9 9 9 15 15 15 F F F 20 400-419 10 10 10 16 16 16 T T T 21 420-439 8 8 8 13 13 13 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 9 9 9 15 15 15 T T T 25 500-519 10 10 10 17 17 17 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 10 10 10 17 17 17 F F F 28 560-579 7 7 7 10 10 10 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 10, Max flush: 17 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 9 9 9 15 15 15 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 11 11 11 18 18 18 F F F 13 260-279 10 10 10 17 17 17 F F F 14 280-299 11 11 11 18 18 18 F F F 15 300-319 7 7 7 10 10 10 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 11 11 11 18 18 18 F F F 18 360-379 10 10 10 17 17 17 F F F 19 380-399 9 9 9 15 15 15 F F F 20 400-419 11 11 11 18 18 18 F F F 21 420-439 8 8 8 13 13 13 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 10 10 10 17 17 17 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 10 10 10 17 17 17 F F F 28 560-579 7 7 7 10 10 10 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 11, Max flush: 18 Begin loop 6 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.514028391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.514349264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.514367361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231} {"msg":"Crucible 511b50a8-a1a4-4ce8-89e4-de6443605a37 has session id: c73f7f28-2837-46e4-a2de-f2698db72edc","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.514685624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Upstairs opts: Upstairs UUID: 511b50a8-a1a4-4ce8-89e4-de6443605a37, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.514732207Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Crucible stats registered with UUID: 511b50a8-a1a4-4ce8-89e4-de6443605a37","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.514756403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515052749Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231} {"msg":"511b50a8-a1a4-4ce8-89e4-de6443605a37 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515145786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515163683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515178601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515192479Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515205997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515219065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515232333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515245351Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515258959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515272527Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:00.515466728Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:00.515498443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:00.515530469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515554265Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515577451Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47159)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515705463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515736248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35286)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.515765244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.516460141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.516485747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.516500165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.516514063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517063471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517179174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517519224Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517538721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517559368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517711505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517738601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:00.517753449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.516470939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45784)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.516579413Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"io task","client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.517637896Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.517665982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518256114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518821201Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518847957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[0]R flush_numbers[0..12]: [2, 8, 2, 13, 13, 10, 4, 18, 18, 18, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518863764Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[0]R generation[0..12]: [1, 6, 1, 8, 8, 7, 3, 11, 11, 11, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518877872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.51889279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[1]R flush_numbers[0..12]: [2, 8, 2, 13, 13, 10, 4, 18, 18, 18, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518907148Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[1]R generation[0..12]: [1, 6, 1, 8, 8, 7, 3, 11, 11, 11, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518924945Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518939393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[2]R flush_numbers[0..12]: [2, 8, 2, 13, 13, 10, 4, 18, 18, 18, 15, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518953081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[2]R generation[0..12]: [1, 6, 1, 8, 8, 7, 3, 11, 11, 11, 9, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518966319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.518980177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:11:10.518993055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519008973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Next flush: 19","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519021891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519034699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"":"downstairs","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519047797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519062495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"0","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519085961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"1","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519110688Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"client":"2","":"downstairs","session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519128065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519140383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"511b50a8-a1a4-4ce8-89e4-de6443605a37 is now active with session: c73f7f28-2837-46e4-a2de-f2698db72edc","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519158831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519185177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231,"session_id":"c73f7f28-2837-46e4-a2de-f2698db72edc"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:10.519207883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 36864 data: 2 3 3 2 2 2 3 3 3 02/30 Write block 542 len 24576 data: 2 2 2 2 3 3 03/30 Read block 334 len 4096 04/30 Write block 348 len 20480 data: 3 3 3 2 2 05/30 Read block 81 len 12288 06/30 Write block 286 len 32768 data: 2 3 3 3 3 3 4 4 07/30 Read block 509 len 12288 08/30 Write block 358 len 16384 data: 2 3 3 3 09/30 Write block 412 len 40960 data: 3 3 3 3 3 3 3 3 3 2 10/30 Flush 11/30 Write block 420 len 24576 data: 4 3 2 2 2 2 12/30 Read block 510 len 16384 13/30 Read block 38 len 40960 14/30 Read block 247 len 36864 15/30 Read block 297 len 28672 16/30 Read block 184 len 8192 17/30 Write block 311 len 28672 data: 3 3 3 3 3 3 3 18/30 Flush 19/30 Write block 514 len 24576 data: 3 3 4 4 4 4 20/30 Read block 219 len 16384 21/30 Write block 206 len 12288 data: 2 2 2 22/30 Flush 23/30 Flush 24/30 Write block 394 len 16384 data: 3 3 4 3 25/30 Read block 88 len 32768 26/30 Read block 150 len 12288 27/30 Write block 206 len 8192 data: 3 3 28/30 Write block 563 len 12288 data: 3 3 3 29/30 Write block 33 len 20480 data: 2 2 2 2 2 30/30 Read block 294 len 8192 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:10.570191536Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1231} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 6 6 6 8 8 8 T T T 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 12 12 12 21 21 21 T T T 11 220-239 9 9 9 15 15 15 F F F 12 240-259 11 11 11 18 18 18 F F F 13 260-279 12 12 12 19 19 19 F F F 14 280-299 12 12 12 19 19 19 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 12 12 12 19 19 19 F F F 18 360-379 12 12 12 19 19 19 F F F 19 380-399 9 9 9 15 15 15 T T T 20 400-419 12 12 12 19 19 19 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 7 7 7 10 10 10 T T T 29 580-599 7 7 7 10 10 10 F F F Max gen: 12, Max flush: 21 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 13 13 13 22 22 22 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 7 7 7 10 10 10 F F F 6 120-139 3 3 3 4 4 4 F F F 7 140-159 11 11 11 18 18 18 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 11 11 11 18 18 18 F F F 13 260-279 12 12 12 19 19 19 F F F 14 280-299 12 12 12 19 19 19 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 12 12 12 19 19 19 F F F 18 360-379 12 12 12 19 19 19 F F F 19 380-399 13 13 13 22 22 22 F F F 20 400-419 12 12 12 19 19 19 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 9 9 9 15 15 15 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 11 11 11 18 18 18 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 10 10 10 17 17 17 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 13 13 13 22 22 22 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 13, Max flush: 22 Begin loop 7 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.93876181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.939202345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.93923368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239} {"msg":"Crucible 4e151061-269d-43e0-ae17-ffefece6907a has session id: b08f1b5a-c408-453f-a07b-5be79e04e10f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.939447998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Upstairs opts: Upstairs UUID: 4e151061-269d-43e0-ae17-ffefece6907a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.939616584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Crucible stats registered with UUID: 4e151061-269d-43e0-ae17-ffefece6907a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.939657887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.939936446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239} {"msg":"4e151061-269d-43e0-ae17-ffefece6907a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940091263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.94011619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940133177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940157164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.94018388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940210136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940235642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940260508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940286824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940310181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50923)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940437122Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:24.940492784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:24.940574382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40986)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940609137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:24.940746486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940771062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.94078521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.940847191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.941278647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.941304183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.941317981Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.941331279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.941781563Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.941875959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.942256412Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.942281689Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.942295636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.94233944Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.942356697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:24.942369985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.942604951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58858)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.942660193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"io task","client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.943640797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.943668773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944273524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944833241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944856727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[0]R flush_numbers[0..12]: [2, 22, 2, 13, 13, 10, 4, 18, 18, 18, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944871935Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[0]R generation[0..12]: [1, 13, 1, 8, 8, 7, 3, 11, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944886143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944900131Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[1]R flush_numbers[0..12]: [2, 22, 2, 13, 13, 10, 4, 18, 18, 18, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944913899Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[1]R generation[0..12]: [1, 13, 1, 8, 8, 7, 3, 11, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944927367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944946034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[2]R flush_numbers[0..12]: [2, 22, 2, 13, 13, 10, 4, 18, 18, 18, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944959822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[2]R generation[0..12]: [1, 13, 1, 8, 8, 7, 3, 11, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.94497313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.944987218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945000156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945013214Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"Wait for a query_work_queue command to finish before sending IO } {"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945029712Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.94504253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"":"downstairs","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945055418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945067356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"0","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945080454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"1","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945093492Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"client":"2","":"downstairs","session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.94510647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945118489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"4e151061-269d-43e0-ae17-ffefece6907a is now active with session: b08f1b5a-c408-453f-a07b-5be79e04e10f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945131107Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945145045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239,"session_id":"b08f1b5a-c408-453f-a07b-5be79e04e10f"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:34.945157263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 441 len 24576 data: 2 4 4 4 4 3 02/30 Write block 342 len 20480 data: 2 2 2 3 3 03/30 Read block 52 len 4096 04/30 Read block 280 len 4096 05/30 Flush 06/30 Write block 337 len 32768 data: 2 2 2 2 2 3 3 3 07/30 Read block 442 len 4096 08/30 Flush 09/30 Write block 520 len 4096 data: 4 10/30 Read block 45 len 28672 11/30 Read block 345 len 4096 12/30 Flush 13/30 Write block 115 len 28672 data: 6 5 4 4 3 3 3 14/30 Read block 0 len 8192 15/30 Read block 466 len 40960 16/30 Read block 118 len 4096 17/30 Write block 112 len 4096 data: 5 18/30 Read block 500 len 36864 19/30 Read block 393 len 28672 20/30 Read block 55 len 20480 21/30 Read block 574 len 8192 22/30 Read block 459 len 4096 23/30 Read block 259 len 36864 24/30 Read block 444 len 32768 25/30 Write block 488 len 16384 data: 6 5 5 4 26/30 Write block 150 len 32768 data: 2 2 2 2 2 2 2 3 27/30 Write block 118 len 40960 data: 5 4 4 4 3 3 2 2 2 2 28/30 Write block 390 len 12288 data: 3 3 3 29/30 Read block 421 len 24576 30/30 Read block 537 len 24576 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 1 2 Done 4 5 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 26 29 28 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:34.970384253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1239} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 13 13 13 22 22 22 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 14 14 14 26 26 26 T T T 6 120-139 14 14 14 26 26 26 T T T 7 140-159 11 11 11 18 18 18 T T T 8 160-179 11 11 11 18 18 18 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 11 11 11 18 18 18 F F F 13 260-279 12 12 12 19 19 19 F F F 14 280-299 12 12 12 19 19 19 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 19 19 19 F F F 19 380-399 13 13 13 22 22 22 T T T 20 400-419 12 12 12 19 19 19 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 14 14 14 26 26 26 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 13 13 13 22 22 22 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 14, Max flush: 26 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 13 13 13 22 22 22 F F F 2 040-059 1 1 1 2 2 2 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 8 8 8 13 13 13 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 11 11 11 18 18 18 F F F 9 180-199 11 11 11 18 18 18 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 9 9 9 15 15 15 F F F 12 240-259 11 11 11 18 18 18 F F F 13 260-279 12 12 12 19 19 19 F F F 14 280-299 12 12 12 19 19 19 F F F 15 300-319 12 12 12 20 20 20 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 12 12 12 19 19 19 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 12 12 12 19 19 19 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 7 7 7 10 10 10 F F F 24 480-499 14 14 14 26 26 26 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 13 13 13 22 22 22 F F F 29 580-599 7 7 7 10 10 10 F F F Max gen: 15, Max flush: 27 Begin loop 8 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.178716439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179073156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179104612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247} {"msg":"Crucible a4a5bcd7-a6a8-42b1-8e76-3ec860251998 has session id: ebf7f003-8790-4c6e-8893-73e34e2049ac","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.17938786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Upstairs opts: Upstairs UUID: a4a5bcd7-a6a8-42b1-8e76-3ec860251998, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179422165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Crucible stats registered with UUID: a4a5bcd7-a6a8-42b1-8e76-3ec860251998","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179441772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179695924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247} {"msg":"a4a5bcd7-a6a8-42b1-8e76-3ec860251998 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.1797902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179812617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179838923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179864019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179897894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.17992217Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179947807Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.179970254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.17999209Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.180015057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:49.180182752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:49.180223986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:35038)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.180253221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:49.180342368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.180370834Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.18039752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:41812)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.180424096Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.180451752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.180986463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.181024987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.181055842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.181082179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.181667402Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.181701617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.182327824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.18235426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.182368898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.182381226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.182397814Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:49.182411072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.181902987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_connection connected from Ok(127.0.0.1:65339)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.181961908Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"io task","client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.183070354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.18309829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.183658387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184220273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184262727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[0]R flush_numbers[0..12]: [2, 22, 2, 13, 13, 27, 27, 27, 18, 18, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184280224Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[0]R generation[0..12]: [1, 13, 1, 8, 8, 15, 15, 15, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184294612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.18430887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[1]R flush_numbers[0..12]: [2, 22, 2, 13, 13, 27, 27, 27, 18, 18, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184322718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[1]R generation[0..12]: [1, 13, 1, 8, 8, 15, 15, 15, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184336266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184350254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[2]R flush_numbers[0..12]: [2, 22, 2, 13, 13, 27, 27, 27, 18, 18, 22, 15]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184368761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[2]R generation[0..12]: [1, 13, 1, 8, 8, 15, 15, 15, 11, 11, 13, 9]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184382549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184396417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184413165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184426253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Next flush: 28","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184438941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184451709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"":"downstairs","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184464937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184476635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"0","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184489813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"1","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184503041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"client":"2","":"downstairs","session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184516169Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184528248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"a4a5bcd7-a6a8-42b1-8e76-3ec860251998 is now active with session: ebf7f003-8790-4c6e-8893-73e34e2049ac","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184540876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184552714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247,"session_id":"ebf7f003-8790-4c6e-8893-73e34e2049ac"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:11:59.184564692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 82 len 36864 data: 3 3 3 2 2 2 2 2 2 02/30 Write block 367 len 36864 data: 2 3 3 3 3 3 4 4 4 03/30 Flush 04/30 Write block 475 len 24576 data: 2 2 2 2 3 3 05/30 Read block 107 len 16384 06/30 Read block 280 len 16384 07/30 Read block 587 len 12288 08/30 Read block 198 len 24576 09/30 Write block 578 len 24576 data: 2 2 2 2 2 2 10/30 Read block 130 len 36864 11/30 Read block 113 len 16384 12/30 Write block 194 len 12288 data: 2 3 3 13/30 Read block 71 len 24576 14/30 Write block 177 len 4096 data: 3 15/30 Write block 160 len 24576 data: 4 4 4 4 4 3 16/30 Write block 42 len 12288 data: 2 2 2 17/30 Read block 361 len 28672 18/30 Read block 381 len 20480 19/30 Write block 236 len 20480 data: 3 2 2 2 2 20/30 Read block 209 len 20480 21/30 Read block 268 len 16384 22/30 Flush 23/30 Write block 257 len 36864 data: 4 4 3 3 3 3 2 2 3 24/30 Write block 21 len 12288 data: 2 2 2 25/30 Write block 576 len 24576 data: 2 2 3 3 3 3 26/30 Write block 295 len 28672 data: 4 3 3 2 3 2 2 27/30 Read block 363 len 24576 28/30 Write block 27 len 28672 data: 3 3 3 2 2 2 3 29/30 Read block 90 len 20480 30/30 Read block 481 len 8192 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:11:59.211533455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1247} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 13 13 13 22 22 22 T T T 2 040-059 16 16 16 29 29 29 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 16 16 16 29 29 29 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 16 16 16 29 29 29 T T T 13 260-279 12 12 12 19 19 19 T T T 14 280-299 12 12 12 19 19 19 T T T 15 300-319 12 12 12 20 20 20 T T T 16 320-339 14 14 14 24 24 24 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 12 12 12 19 19 19 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 16 16 16 29 29 29 T T T 29 580-599 16 16 16 29 29 29 T T T Max gen: 16, Max flush: 29 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 17 17 17 30 30 30 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 15 15 15 27 27 27 F F F 7 140-159 15 15 15 27 27 27 F F F 8 160-179 16 16 16 29 29 29 F F F 9 180-199 16 16 16 29 29 29 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 17 17 17 30 30 30 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 17 17 17 30 30 30 F F F 15 300-319 17 17 17 30 30 30 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 12 12 12 19 19 19 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 16 16 16 29 29 29 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 14 14 14 25 25 25 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 17 17 17 30 30 30 F F F 29 580-599 17 17 17 30 30 30 F F F Max gen: 17, Max flush: 30 Begin loop 9 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.465885427Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.466462071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.466490437Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255} {"msg":"Crucible 8503eb41-857e-440b-967c-5b27d2c0faf8 has session id: 9feb5acd-8a55-4f2d-9753-d6dad9c10639","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.466649444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Upstairs opts: Upstairs UUID: 8503eb41-857e-440b-967c-5b27d2c0faf8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.466696707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Crucible stats registered with UUID: 8503eb41-857e-440b-967c-5b27d2c0faf8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.466726123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467055754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255} {"msg":"8503eb41-857e-440b-967c-5b27d2c0faf8 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467140731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467159028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467174166Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467187884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467201282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.46721436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467227948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467240736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467255144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467268892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_connection connected from Ok(127.0.0.1:42413)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467499808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:13.467539482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:13.467567248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:13.467598973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467663304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467690269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_connection connected from Ok(127.0.0.1:44913)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467712726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.467807492Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.468343703Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.468371459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.468386926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.468408073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469045828Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469073255Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469621443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469647769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469662037Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469701771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469718659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:13.469735366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.468713108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_connection connected from Ok(127.0.0.1:62132)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.468770049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"io task","client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.469796767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.469824313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.470463709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471067549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471095145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[0]R flush_numbers[0..12]: [2, 30, 29, 13, 28, 27, 27, 27, 29, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471111642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[0]R generation[0..12]: [1, 17, 16, 8, 16, 15, 15, 15, 16, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.47112608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471140848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[1]R flush_numbers[0..12]: [2, 30, 29, 13, 28, 27, 27, 27, 29, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471154796Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[1]R generation[0..12]: [1, 17, 16, 8, 16, 15, 15, 15, 16, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471168424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471182262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[2]R flush_numbers[0..12]: [2, 30, 29, 13, 28, 27, 27, 27, 29, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471200449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[2]R generation[0..12]: [1, 17, 16, 8, 16, 15, 15, 15, 16, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471214327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:12:23.471228885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471243753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471257031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Next flush: 31","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471269529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471282277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"":"downstairs","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471295295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471307313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"0","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471320371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"1","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.47133328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"client":"2","":"downstairs","session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471346448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471358646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"8503eb41-857e-440b-967c-5b27d2c0faf8 is now active with session: 9feb5acd-8a55-4f2d-9753-d6dad9c10639","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471371364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471382972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255,"session_id":"9feb5acd-8a55-4f2d-9753-d6dad9c10639"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:23.471395031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 40960 data: 2 2 2 2 2 2 2 2 2 2 02/30 Read block 58 len 36864 03/30 Flush 04/30 Write block 489 len 8192 data: 6 6 05/30 Write block 360 len 20480 data: 4 4 3 2 2 06/30 Read block 434 len 20480 07/30 Write block 563 len 4096 data: 4 08/30 Read block 348 len 40960 09/30 Read block 20 len 16384 10/30 Read block 170 len 32768 11/30 Read block 400 len 8192 12/30 Write block 156 len 20480 data: 3 4 4 4 5 13/30 Read block 247 len 28672 14/30 Flush 15/30 Write block 595 len 12288 data: 3 3 2 16/30 Read block 449 len 40960 17/30 Read block 388 len 28672 18/30 Write block 281 len 16384 data: 2 3 2 2 19/30 Read block 426 len 16384 20/30 Read block 260 len 20480 21/30 Read block 501 len 8192 22/30 Write block 521 len 16384 data: 4 3 2 2 23/30 Read block 184 len 16384 24/30 Read block 53 len 32768 25/30 Read block 472 len 20480 26/30 Write block 162 len 28672 data: 5 5 5 4 2 2 2 27/30 Read block 231 len 32768 28/30 Read block 592 len 28672 29/30 Read block 329 len 4096 30/30 Write block 409 len 16384 data: 4 4 4 4 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 7 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false 31 Acked 1030 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:23.498901973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1255} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 17 17 17 30 30 30 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 18 18 18 32 32 32 F F F 8 160-179 18 18 18 32 32 32 T T T 9 180-199 16 16 16 29 29 29 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 17 17 17 30 30 30 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 17 17 17 30 30 30 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 12 12 12 19 19 19 T T T 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 14 14 14 25 25 25 T T T 27 540-559 12 12 12 19 19 19 F F F 28 560-579 18 18 18 32 32 32 F F F 29 580-599 18 18 18 33 33 33 F F F Max gen: 18, Max flush: 33 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 17 17 17 30 30 30 F F F 2 040-059 16 16 16 29 29 29 F F F 3 060-079 8 8 8 13 13 13 F F F 4 080-099 16 16 16 28 28 28 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 18 18 18 32 32 32 F F F 8 160-179 19 19 19 34 34 34 F F F 9 180-199 16 16 16 29 29 29 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 17 17 17 30 30 30 F F F 13 260-279 17 17 17 30 30 30 F F F 14 280-299 18 18 18 33 33 33 F F F 15 300-319 17 17 17 30 30 30 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 14 14 14 24 24 24 F F F 18 360-379 18 18 18 32 32 32 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 12 12 12 20 20 20 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 19 19 19 34 34 34 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 18 18 18 32 32 32 F F F 29 580-599 18 18 18 33 33 33 F F F Max gen: 19, Max flush: 34 Begin loop 10 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.693764076Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.694091388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.694113825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263} {"msg":"Crucible 954ac413-c5a3-4a08-a631-0a0c5624d526 has session id: bb23dfa6-cce5-4a91-9c19-8f5f2799a311","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.694520584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Upstairs opts: Upstairs UUID: 954ac413-c5a3-4a08-a631-0a0c5624d526, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.694565098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Crucible stats registered with UUID: 954ac413-c5a3-4a08-a631-0a0c5624d526","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.694584505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.694969638Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263} {"msg":"954ac413-c5a3-4a08-a631-0a0c5624d526 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695032208Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695059115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695075562Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.69508939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695103358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695125905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695147622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.69516136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695175977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695189565Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_connection connected from Ok(127.0.0.1:63318)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695380417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_connection connected from Ok(127.0.0.1:53141)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695525715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:37.695581327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:37.6956311Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:37.695716457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695736954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695755691Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.695814293Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.696198066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.696222822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.69623783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.696251628Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.696850299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.696877145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.697459949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.697498493Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.697526609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.697540957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.697554755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:37.697571242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.697824305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_connection connected from Ok(127.0.0.1:44021)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.697915711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"io task","client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.698850483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.698878598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.699457483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700048545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700079031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[0]R flush_numbers[0..12]: [2, 30, 29, 13, 28, 27, 31, 32, 34, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700094218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[0]R generation[0..12]: [1, 17, 16, 8, 16, 15, 18, 18, 19, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700108446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700122694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[1]R flush_numbers[0..12]: [2, 30, 29, 13, 28, 27, 31, 32, 34, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700136342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[1]R generation[0..12]: [1, 17, 16, 8, 16, 15, 18, 18, 19, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.7001499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700164258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[2]R flush_numbers[0..12]: [2, 30, 29, 13, 28, 27, 31, 32, 34, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700178016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[2]R generation[0..12]: [1, 17, 16, 8, 16, 15, 18, 18, 19, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700196683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700211131Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700224139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700237177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Next flush: 35","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:12:47.700249855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700264383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"":"downstairs","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700277551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700289399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"0","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700302598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"1","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700315416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"client":"2","":"downstairs","session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700328623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700340822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"954ac413-c5a3-4a08-a631-0a0c5624d526 is now active with session: bb23dfa6-cce5-4a91-9c19-8f5f2799a311","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.70035365Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700365578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263,"session_id":"bb23dfa6-cce5-4a91-9c19-8f5f2799a311"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:12:47.700377576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 437 len 8192 data: 2 2 02/30 Write block 57 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Write block 586 len 28672 data: 2 2 2 2 2 2 2 04/30 Write block 351 len 20480 data: 3 3 2 2 2 05/30 Read block 345 len 32768 06/30 Read block 207 len 32768 07/30 Read block 567 len 16384 08/30 Flush 09/30 Write block 55 len 24576 data: 2 2 3 3 3 3 10/30 Read block 365 len 20480 11/30 Read block 536 len 12288 12/30 Flush 13/30 Write block 156 len 4096 data: 4 14/30 Write block 299 len 32768 data: 4 3 3 2 2 2 2 2 15/30 Read block 259 len 24576 16/30 Write block 351 len 40960 data: 4 4 3 3 3 2 2 3 4 5 17/30 Read block 26 len 4096 18/30 Write block 148 len 40960 data: 2 2 3 3 3 3 3 3 5 5 19/30 Read block 365 len 24576 20/30 Read block 445 len 28672 21/30 Read block 550 len 8192 22/30 Read block 151 len 32768 23/30 Flush 24/30 Write block 157 len 4096 data: 6 25/30 Read block 309 len 28672 26/30 Read block 280 len 12288 27/30 Read block 158 len 12288 28/30 Write block 77 len 16384 data: 2 4 4 4 29/30 Read block 556 len 20480 30/30 Write block 274 len 28672 data: 3 3 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:12:47.728305027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1263} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 17 17 17 30 30 30 F F F 2 040-059 20 20 20 36 36 36 F F F 3 060-079 20 20 20 36 36 36 T T T 4 080-099 16 16 16 28 28 28 T T T 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 20 20 20 38 38 38 T T T 8 160-179 19 19 19 34 34 34 F F F 9 180-199 16 16 16 29 29 29 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 17 17 17 30 30 30 F F F 13 260-279 17 17 17 30 30 30 T T T 14 280-299 20 20 20 37 37 37 T T T 15 300-319 20 20 20 37 37 37 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 20 20 20 37 37 37 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 19 19 19 34 34 34 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 18 18 18 32 32 32 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 20, Max flush: 38 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 F F F 1 020-039 17 17 17 30 30 30 F F F 2 040-059 20 20 20 36 36 36 F F F 3 060-079 21 21 21 39 39 39 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 21 21 21 39 39 39 F F F 8 160-179 19 19 19 34 34 34 F F F 9 180-199 16 16 16 29 29 29 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 16 16 16 29 29 29 F F F 12 240-259 17 17 17 30 30 30 F F F 13 260-279 21 21 21 39 39 39 F F F 14 280-299 21 21 21 39 39 39 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 20 20 20 37 37 37 F F F 18 360-379 20 20 20 37 37 37 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 12 12 12 21 21 21 F F F 26 520-539 19 19 19 34 34 34 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 18 18 18 32 32 32 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 21, Max flush: 39 Begin loop 11 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.969064877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.969687644Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.969721409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271} {"msg":"Crucible d2dbbfbf-8159-41d9-9fbc-80cd3b0a1b03 has session id: ced3d807-6663-4d50-bb7c-c90c57b3b3d1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.969745556Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Upstairs opts: Upstairs UUID: d2dbbfbf-8159-41d9-9fbc-80cd3b0a1b03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.969806587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Crucible stats registered with UUID: d2dbbfbf-8159-41d9-9fbc-80cd3b0a1b03","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.969831063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.97018609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271} {"msg":"d2dbbfbf-8159-41d9-9fbc-80cd3b0a1b03 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970218136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970232204Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970247571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970270738Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970296904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970320511Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970343467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970367733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.97039079Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970414177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54149)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970563424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:01.970600659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:01.970628455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:34942)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970654871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:01.970739858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970771564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.97079862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.970940908Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.971436595Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.971476469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.971492907Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.971506545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972127393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972255584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972712896Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972735613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.97274956Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972835098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972853345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:01.972866773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.97194524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:50492)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.97201331Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"io task","client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.972987895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.973028029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.973661765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974296661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974330206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[0]R flush_numbers[0..12]: [2, 30, 36, 39, 39, 27, 31, 39, 34, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974346114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[0]R generation[0..12]: [1, 17, 20, 21, 21, 15, 18, 21, 19, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.97444201Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974461317Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[1]R flush_numbers[0..12]: [2, 30, 36, 39, 39, 27, 31, 39, 34, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974475275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[1]R generation[0..12]: [1, 17, 20, 21, 21, 15, 18, 21, 19, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974488763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974502671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[2]R flush_numbers[0..12]: [2, 30, 36, 39, 39, 27, 31, 39, 34, 29, 22, 29]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974516159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[2]R generation[0..12]: [1, 17, 20, 21, 21, 15, 18, 21, 19, 16, 13, 16]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974533936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974548444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974561392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.97457454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Next flush: 40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974587298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974600156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"":"downstairs","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974613174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974624782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"0","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974638011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"1","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Transition from WaitQuorum to Active"Wait for a query_work_queue command to finish before sending IO ,"v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974654408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"client":"2","":"downstairs","session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974667736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974679674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"d2dbbfbf-8159-41d9-9fbc-80cd3b0a1b03 is now active with session: ced3d807-6663-4d50-bb7c-c90c57b3b3d1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974692232Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974704131Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271,"session_id":"ced3d807-6663-4d50-bb7c-c90c57b3b3d1"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:11.974716009Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 294 len 16384 data: 4 5 4 4 02/30 Write block 143 len 28672 data: 3 3 3 2 2 3 3 03/30 Read block 267 len 40960 04/30 Read block 509 len 8192 05/30 Write block 514 len 8192 data: 4 4 06/30 Read block 57 len 24576 07/30 Write block 359 len 12288 data: 5 6 5 08/30 Write block 280 len 36864 data: 3 3 4 3 3 2 3 4 4 09/30 Read block 364 len 28672 10/30 Read block 12 len 28672 11/30 Write block 199 len 28672 data: 4 4 4 4 4 3 3 12/30 Read block 258 len 12288 13/30 Read block 155 len 24576 14/30 Write block 20 len 4096 data: 2 15/30 Write block 238 len 12288 data: 3 3 3 16/30 Read block 155 len 32768 17/30 Read block 121 len 12288 18/30 Write block 450 len 32768 data: 4 3 3 3 3 3 3 3 19/30 Write block 261 len 36864 data: 4 4 3 3 4 4 4 3 3 20/30 Read block 54 len 24576 21/30 Write block 0 len 16384 data: 2 2 2 2 22/30 Read block 392 len 8192 23/30 Write block 573 len 12288 data: 2 2 2 24/30 Read block 296 len 20480 25/30 Read block 372 len 12288 26/30 Read block 91 len 36864 27/30 Read block 79 len 16384 28/30 Read block 526 len 4096 29/30 Read block 571 len 28672 30/30 Write block 50 len 28672 data: 2 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 3 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Read 7 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:11.999303715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1271} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 2 2 2 T T T 1 020-039 22 22 22 40 40 40 F F F 2 040-059 20 20 20 36 36 36 T T T 3 060-079 21 21 21 39 39 39 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 19 19 19 34 34 34 F F F 9 180-199 22 22 22 40 40 40 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 22 22 22 40 40 40 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 22 22 22 40 40 40 F F F 26 520-539 19 19 19 34 34 34 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 18 18 18 32 32 32 T T T 29 580-599 20 20 20 35 35 35 F F F Max gen: 22, Max flush: 40 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 41 41 41 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 21 21 21 39 39 39 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 19 19 19 34 34 34 F F F 9 180-199 22 22 22 40 40 40 F F F 10 200-219 22 22 22 40 40 40 F F F 11 220-239 22 22 22 40 40 40 F F F 12 240-259 22 22 22 40 40 40 F F F 13 260-279 22 22 22 40 40 40 F F F 14 280-299 22 22 22 40 40 40 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 40 40 40 F F F 23 460-479 16 16 16 29 29 29 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 22 22 22 40 40 40 F F F 26 520-539 19 19 19 34 34 34 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 23 23 23 41 41 41 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.194571707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195001973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195032959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279} {"msg":"Crucible 54118881-4177-4a43-b56f-3b3b09a6e29a has session id: 2b114902-01b8-4900-b290-d575c38b7d13","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195130674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Upstairs opts: Upstairs UUID: 54118881-4177-4a43-b56f-3b3b09a6e29a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195169268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Crucible stats registered with UUID: 54118881-4177-4a43-b56f-3b3b09a6e29a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195192695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195505278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279} {"msg":"54118881-4177-4a43-b56f-3b3b09a6e29a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195535004Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195549982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.19556515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195578907Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195592316Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195605314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195618672Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195667425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.19569831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.195724216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_connection connected from Ok(127.0.0.1:34384)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.19596491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_connection connected from Ok(127.0.0.1:42201)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196287993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:26.196330816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:26.196350733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:26.196420403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.19643821Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196451598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196566761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196828142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196854798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196964822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.196986399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.197398658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.197531768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.197901993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.197932419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.197955905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.198035363Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.19805524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:26.198072358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.198553397Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_connection connected from Ok(127.0.0.1:51336)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.198707314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"io task","client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.199641855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.199669351Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200260264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200806372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200834168Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[0]R flush_numbers[0..12]: [41, 40, 41, 39, 39, 27, 31, 40, 34, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200850766Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[0]R generation[0..12]: [23, 22, 23, 21, 21, 15, 18, 22, 19, 22, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200865194Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200879452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[1]R flush_numbers[0..12]: [41, 40, 41, 39, 39, 27, 31, 40, 34, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.20089322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[1]R generation[0..12]: [23, 22, 23, 21, 21, 15, 18, 22, 19, 22, 22, 22]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200906937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200920715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[2]R flush_numbers[0..12]: [41, 40, 41, 39, 39, 27, 31, 40, 34, 40, 40, 40]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200934364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[2]R generation[0..12]: [23, 22, 23, 21, 21, 15, 18, 22, 19, 22, 22, 22]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:13:36.200952511Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200968938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200981997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.200994924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201007773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201020571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"":"downstairs","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201033509Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201045407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"0","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201058385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"1","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201071263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"client":"2","":"downstairs","session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201084391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.20109648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"54118881-4177-4a43-b56f-3b3b09a6e29a is now active with session: 2b114902-01b8-4900-b290-d575c38b7d13","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201109158Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201121116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279,"session_id":"2b114902-01b8-4900-b290-d575c38b7d13"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:36.201133234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 572 len 20480 data: 3 3 3 3 3 02/30 Read block 83 len 20480 03/30 Write block 504 len 24576 data: 3 3 3 3 3 3 04/30 Read block 208 len 12288 05/30 Read block 287 len 32768 06/30 Write block 166 len 8192 data: 3 3 07/30 Read block 113 len 40960 08/30 Read block 593 len 8192 09/30 Write block 217 len 16384 data: 3 3 4 4 10/30 Write block 242 len 20480 data: 3 2 2 2 2 11/30 Write block 274 len 20480 data: 4 4 3 3 3 12/30 Read block 319 len 36864 13/30 Write block 517 len 20480 data: 5 5 5 5 5 14/30 Flush 15/30 Write block 169 len 28672 data: 2 2 2 2 3 3 3 16/30 Read block 143 len 24576 17/30 Write block 453 len 36864 data: 4 4 4 4 4 4 4 4 4 18/30 Write block 176 len 24576 data: 3 4 3 3 3 2 19/30 Read block 361 len 32768 20/30 Write block 9 len 16384 data: 2 2 2 2 21/30 Write block 283 len 20480 data: 4 4 3 4 5 22/30 Write block 532 len 24576 data: 2 2 2 2 2 2 23/30 Write block 160 len 20480 data: 6 5 6 6 6 24/30 Read block 306 len 12288 25/30 Write block 8 len 36864 data: 2 3 3 3 3 2 2 2 2 26/30 Read block 173 len 28672 27/30 Write block 212 len 24576 data: 2 3 3 3 3 4 28/30 Read block 58 len 36864 29/30 Write block 467 len 40960 data: 4 2 2 2 2 2 2 2 3 3 30/30 Read block 392 len 36864 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Read 8 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 10 Done Done Done false 31 Acked 1030 Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:36.225744526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1279} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 23 23 23 41 41 41 T T T 1 020-039 22 22 22 40 40 40 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 21 21 21 39 39 39 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 24 24 24 43 43 43 T T T 9 180-199 22 22 22 40 40 40 T T T 10 200-219 24 24 24 42 42 42 T T T 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 22 22 22 40 40 40 T T T 15 300-319 20 20 20 37 37 37 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 22 22 22 40 40 40 T T T 23 460-479 16 16 16 29 29 29 T T T 24 480-499 18 18 18 32 32 32 F F F 25 500-519 24 24 24 42 42 42 F F F 26 520-539 24 24 24 42 42 42 T T T 27 540-559 12 12 12 19 19 19 F F F 28 560-579 24 24 24 42 42 42 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 24, Max flush: 43 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 44 44 44 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 21 21 21 39 39 39 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 15 15 15 27 27 27 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 25 25 25 44 44 44 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 25 25 25 44 44 44 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 14 14 14 24 24 24 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 15 15 15 27 27 27 F F F 20 400-419 19 19 19 34 34 34 F F F 21 420-439 20 20 20 35 35 35 F F F 22 440-459 25 25 25 44 44 44 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 18 18 18 32 32 32 F F F 25 500-519 24 24 24 42 42 42 F F F 26 520-539 25 25 25 44 44 44 F F F 27 540-559 12 12 12 19 19 19 F F F 28 560-579 24 24 24 42 42 42 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 25, Max flush: 44 Begin loop 13 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.474401625Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.47490652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.474935246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287} {"msg":"Crucible a633b8b7-221e-4184-8873-7c7d5976cfc6 has session id: 931d9bd5-6296-42c9-9c92-9a61a6290c59","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.474957153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Upstairs opts: Upstairs UUID: a633b8b7-221e-4184-8873-7c7d5976cfc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475023503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Crucible stats registered with UUID: a633b8b7-221e-4184-8873-7c7d5976cfc6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475046439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475391008Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287} {"msg":"a633b8b7-221e-4184-8873-7c7d5976cfc6 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475421084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.47544526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475471907Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475494693Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.47551852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475542806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475568272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475593449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.475621154Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.47564792Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:50.475811386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:50.475933348Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:13:50.475976212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476002608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476027004Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_connection connected from Ok(127.0.0.1:52661)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.47605337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_connection connected from Ok(127.0.0.1:50527)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476078777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476104303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476657451Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476685087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476699885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.476713323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477215178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477318583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477717934Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477756698Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477785024Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477846744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.4778769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:13:50.477892598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.477634276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_connection connected from Ok(127.0.0.1:41667)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.47787598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"io task","client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.478791554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.47881923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.479370789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.479945423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.479973529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[0]R flush_numbers[0..12]: [44, 40, 41, 39, 39, 27, 31, 40, 44, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.479990367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[0]R generation[0..12]: [25, 22, 23, 21, 21, 15, 18, 22, 25, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480004784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480019182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[1]R flush_numbers[0..12]: [44, 40, 41, 39, 39, 27, 31, 40, 44, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.48003283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[1]R generation[0..12]: [25, 22, 23, 21, 21, 15, 18, 22, 25, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480046678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480065766Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[2]R flush_numbers[0..12]: [44, 40, 41, 39, 39, 27, 31, 40, 44, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480079853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"[2]R generation[0..12]: [25, 22, 23, 21, 21, 15, 18, 22, 25, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480093691Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480111069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480124307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480137385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Next flush: 45","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480150073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480162891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"":"downstairs","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480175779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480187597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"0","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480200675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"1","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480213693Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"client":"2","":"downstairs","session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480226842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.48023887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"a633b8b7-221e-4184-8873-7c7d5976cfc6 is now active with session: 931d9bd5-6296-42c9-9c92-9a61a6290c59","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480251598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480265166Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287,"session_id":"931d9bd5-6296-42c9-9c92-9a61a6290c59"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:00.480277374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 427 len 28672 data: 3 3 3 3 3 3 3 02/30 Flush 03/30 Write block 434 len 40960 data: 2 2 2 3 3 2 2 3 5 5 04/30 Write block 388 len 8192 data: 3 3 05/30 Write block 488 len 40960 data: 7 7 7 5 3 2 2 2 2 3 06/30 Write block 162 len 32768 data: 7 7 7 5 4 4 3 3 07/30 Read block 323 len 4096 08/30 Flush 09/30 Write block 394 len 12288 data: 4 4 5 10/30 Read block 105 len 40960 11/30 Read block 190 len 12288 12/30 Write block 44 len 8192 data: 3 2 13/30 Flush 14/30 Write block 570 len 8192 data: 2 4 15/30 Read block 230 len 16384 16/30 Read block 380 len 8192 17/30 Read block 361 len 8192 18/30 Write block 104 len 24576 data: 3 3 3 3 4 4 19/30 Write block 539 len 8192 data: 2 2 20/30 Read block 323 len 20480 21/30 Read block 30 len 24576 22/30 Write block 333 len 28672 data: 2 2 2 2 3 3 3 23/30 Flush 24/30 Write block 431 len 16384 data: 4 4 4 3 25/30 Read block 404 len 40960 26/30 Write block 402 len 40960 data: 4 4 5 5 6 6 5 5 5 5 27/30 Write block 65 len 20480 data: 3 2 2 2 2 28/30 Write block 134 len 16384 data: 3 3 3 3 29/30 Read block 45 len 4096 30/30 Write block 498 len 12288 data: 3 2 2 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:00.508555972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1287} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 44 44 44 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 21 21 21 39 39 39 T T T 4 080-099 21 21 21 39 39 39 F F F 5 100-119 26 26 26 48 48 48 F F F 6 120-139 18 18 18 31 31 31 T T T 7 140-159 22 22 22 40 40 40 F F F 8 160-179 26 26 26 46 46 46 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 25 25 25 44 44 44 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 19 19 19 34 34 34 T T T 21 420-439 26 26 26 46 46 46 T T T 22 440-459 26 26 26 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 26 26 26 46 46 46 T T T 25 500-519 24 24 24 42 42 42 T T T 26 520-539 26 26 26 49 49 49 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 26 26 26 48 48 48 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 26, Max flush: 49 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 44 44 44 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 26 26 26 47 47 47 F F F 3 060-079 27 27 27 50 50 50 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 26 26 26 48 48 48 F F F 6 120-139 27 27 27 50 50 50 F F F 7 140-159 22 22 22 40 40 40 F F F 8 160-179 26 26 26 46 46 46 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 25 25 25 44 44 44 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 22 22 22 40 40 40 F F F 18 360-379 22 22 22 40 40 40 F F F 19 380-399 26 26 26 47 47 47 F F F 20 400-419 27 27 27 50 50 50 F F F 21 420-439 27 27 27 50 50 50 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 27 27 27 50 50 50 F F F 25 500-519 27 27 27 50 50 50 F F F 26 520-539 26 26 26 49 49 49 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 26 26 26 48 48 48 F F F 29 580-599 20 20 20 35 35 35 F F F Max gen: 27, Max flush: 50 Begin loop 14 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.738654642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739384534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.73941023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295} {"msg":"Crucible 3ca7c0ea-f58d-4c52-8243-3c8ef6056e72 has session id: 9fd91a22-5923-44c5-b047-fdf0092ab648","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739433916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Upstairs opts: Upstairs UUID: 3ca7c0ea-f58d-4c52-8243-3c8ef6056e72, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739470421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Crucible stats registered with UUID: 3ca7c0ea-f58d-4c52-8243-3c8ef6056e72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739501327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739807371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295} {"msg":"3ca7c0ea-f58d-4c52-8243-3c8ef6056e72 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739873681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739899757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739915985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739938062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739965458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.739991784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.74001833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.740044666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.740071182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.740095288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:14.740200733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:14.740238327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:14.740264593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.74028983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.740311546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_connection connected from Ok(127.0.0.1:35201)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.740380336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.740410572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_connection connected from Ok(127.0.0.1:55623)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.74048549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.741072694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.741110478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.741141123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.741168349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.741780309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.741807494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.742351684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.74237817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.742392108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.742404186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.742420934Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:14.742434172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.74103247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_connection connected from Ok(127.0.0.1:43554)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.741184547Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"io task","client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.742224753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.742263097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.742843921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743381001Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743402568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[0]R flush_numbers[0..12]: [44, 40, 47, 50, 39, 48, 50, 40, 46, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743425185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[0]R generation[0..12]: [25, 22, 26, 27, 21, 26, 27, 22, 26, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743439763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743454121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[1]R flush_numbers[0..12]: [44, 40, 47, 50, 39, 48, 50, 40, 46, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743467859Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[1]R generation[0..12]: [25, 22, 26, 27, 21, 26, 27, 22, 26, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743481577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743551496Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[2]R flush_numbers[0..12]: [44, 40, 47, 50, 39, 48, 50, 40, 46, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743574523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[2]R generation[0..12]: [25, 22, 26, 27, 21, 26, 27, 22, 26, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743588811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743602919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743615787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743628635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743641343Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg"Wait for a query_work_queue command to finish before sending IO :"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743658111Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"":"downstairs","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743671298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743683207Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"0","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743696695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"1","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743709793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"client":"2","":"downstairs","session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743723011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743735189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"3ca7c0ea-f58d-4c52-8243-3c8ef6056e72 is now active with session: 9fd91a22-5923-44c5-b047-fdf0092ab648","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743747797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743759615Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295,"session_id":"9fd91a22-5923-44c5-b047-fdf0092ab648"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:24.743773783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 365 len 40960 data: 2 2 3 4 4 4 4 4 5 5 02/30 Write block 401 len 20480 data: 4 5 5 6 6 03/30 Write block 598 len 4096 data: 2 04/30 Read block 111 len 40960 05/30 Write block 345 len 24576 data: 4 4 2 4 4 4 06/30 Read block 581 len 36864 07/30 Read block 370 len 28672 08/30 Read block 168 len 8192 09/30 Flush 10/30 Write block 562 len 40960 data: 3 5 4 4 2 2 2 2 3 5 11/30 Flush 12/30 Flush 13/30 Write block 380 len 32768 data: 4 3 3 3 3 2 2 2 14/30 Read block 5 len 28672 15/30 Write block 507 len 40960 data: 4 4 4 3 3 3 2 5 5 5 16/30 Read block 587 len 24576 17/30 Read block 138 len 8192 18/30 Read block 496 len 16384 19/30 Write block 113 len 4096 data: 5 20/30 Read block 229 len 24576 21/30 Read block 159 len 12288 22/30 Read block 499 len 24576 23/30 Read block 112 len 36864 24/30 Write block 385 len 28672 data: 3 3 3 4 4 4 4 25/30 Read block 265 len 8192 26/30 Read block 87 len 8192 27/30 Read block 399 len 12288 28/30 Write block 416 len 8192 data: 4 4 29/30 Write block 153 len 16384 data: 4 4 4 6 30/30 Write block 49 len 36864 data: 2 3 3 3 3 3 4 4 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Write 4 Done Done Done false 31 Acked 1030 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:24.769750993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1295} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 44 44 44 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 26 26 26 47 47 47 T T T 3 060-079 27 27 27 50 50 50 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 28 28 28 54 54 54 F F F 6 120-139 27 27 27 50 50 50 F F F 7 140-159 22 22 22 40 40 40 T T T 8 160-179 26 26 26 46 46 46 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 25 25 25 44 44 44 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 28 28 28 51 51 51 F F F 18 360-379 28 28 28 51 51 51 F F F 19 380-399 28 28 28 54 54 54 T T T 20 400-419 28 28 28 51 51 51 T T T 21 420-439 27 27 27 50 50 50 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 27 27 27 50 50 50 F F F 25 500-519 28 28 28 54 54 54 F F F 26 520-539 26 26 26 49 49 49 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 28 28 28 52 52 52 F F F 29 580-599 28 28 28 51 51 51 F F F Max gen: 28, Max flush: 54 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 44 44 44 F F F 1 020-039 22 22 22 40 40 40 F F F 2 040-059 29 29 29 55 55 55 F F F 3 060-079 27 27 27 50 50 50 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 28 28 28 54 54 54 F F F 6 120-139 27 27 27 50 50 50 F F F 7 140-159 29 29 29 55 55 55 F F F 8 160-179 26 26 26 46 46 46 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 24 24 24 42 42 42 F F F 12 240-259 24 24 24 42 42 42 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 25 25 25 44 44 44 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 28 28 28 51 51 51 F F F 18 360-379 28 28 28 51 51 51 F F F 19 380-399 29 29 29 55 55 55 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 27 27 27 50 50 50 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 25 25 25 44 44 44 F F F 24 480-499 27 27 27 50 50 50 F F F 25 500-519 28 28 28 54 54 54 F F F 26 520-539 26 26 26 49 49 49 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 28 28 28 52 52 52 F F F 29 580-599 28 28 28 51 51 51 F F F Max gen: 29, Max flush: 55 Begin loop 15 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.979404591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.979665892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.979694318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303} {"msg":"Crucible 36eb3cf2-3589-4fa7-9a0b-64cf211cf631 has session id: f90ed23a-b54b-4496-86ac-4d3cab1769d1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980136093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Upstairs opts: Upstairs UUID: 36eb3cf2-3589-4fa7-9a0b-64cf211cf631, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980168948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Crucible stats registered with UUID: 36eb3cf2-3589-4fa7-9a0b-64cf211cf631","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980187945Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980606793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303} {"msg":"36eb3cf2-3589-4fa7-9a0b-64cf211cf631 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980645887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980660525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980675023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980688211Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980701609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980722895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980748312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980774398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.980800884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.98082621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:38.980979568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:53276)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981015502Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:38.981044698Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:38.981114937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981145673Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981160841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45401)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981251477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981319877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981766151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981793157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981808605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.981829911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982303281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982417954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982779861Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982810986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982834813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982922169Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982943886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:38.982957494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.981818084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_connection connected from Ok(127.0.0.1:51384)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.98191337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"io task","client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.982821315Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.98285487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.983447182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984040074Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.98406816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[0]R flush_numbers[0..12]: [44, 40, 55, 50, 39, 54, 50, 55, 46, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984093326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[0]R generation[0..12]: [25, 22, 29, 27, 21, 28, 27, 29, 26, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984108194Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984122472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[1]R flush_numbers[0..12]: [44, 40, 55, 50, 39, 54, 50, 55, 46, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.98413608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[1]R generation[0..12]: [25, 22, 29, 27, 21, 28, 27, 29, 26, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984149628Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984163546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[2]R flush_numbers[0..12]: [44, 40, 55, 50, 39, 54, 50, 55, 46, 44, 44, 42]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984182463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[2]R generation[0..12]: [25, 22, 29, 27, 21, 28, 27, 29, 26, 25, 25, 24]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984196511Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984210519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984223347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984240425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Next flush: 56","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984253383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984266231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"":"downstairs","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984279259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984290967Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"0","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984304055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"1","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984317073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"client":"2","":"downstairs","session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984330171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984342119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"36eb3cf2-3589-4fa7-9a0b-64cf211cf631 is now active with session: f90ed23a-b54b-4496-86ac-4d3cab1769d1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984354808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984367226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303,"session_id":"f90ed23a-b54b-4496-86ac-4d3cab1769d1"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:14:48.984381953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 36864 data: 5 6 6 5 5 5 5 5 4 02/30 Read block 365 len 8192 03/30 Write block 290 len 4096 data: 4 04/30 Read block 256 len 16384 05/30 Read block 8 len 4096 06/30 Write block 287 len 36864 data: 6 5 4 5 4 5 5 5 6 07/30 Write block 428 len 36864 data: 4 4 4 5 5 5 4 3 3 08/30 Read block 73 len 36864 09/30 Read block 555 len 8192 10/30 Write block 580 len 8192 data: 4 4 11/30 Write block 531 len 36864 data: 2 3 3 3 3 3 3 2 3 12/30 Read block 558 len 40960 13/30 Read block 267 len 16384 14/30 Read block 351 len 32768 15/30 Write block 15 len 40960 data: 3 3 2 2 2 3 3 3 3 3 16/30 Write block 476 len 36864 data: 4 3 3 4 4 3 3 3 5 17/30 Flush 18/30 Write block 129 len 4096 data: 2 19/30 Read block 478 len 24576 20/30 Read block 568 len 40960 21/30 Read block 429 len 12288 22/30 Read block 178 len 24576 23/30 Read block 165 len 8192 24/30 Read block 218 len 36864 25/30 Read block 444 len 40960 26/30 Write block 113 len 4096 data: 6 27/30 Read block 573 len 32768 28/30 Read block 396 len 8192 29/30 Write block 154 len 32768 data: 5 5 7 7 5 5 7 6 30/30 Write block 239 len 12288 data: 4 4 2 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 3 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Read 10 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 8 Done Sent Sent false 30 Acked 1029 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 2 2 5 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:14:49.009808805Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1303} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 56 56 56 F F F 1 020-039 30 30 30 56 56 56 F F F 2 040-059 29 29 29 55 55 55 F F F 3 060-079 27 27 27 50 50 50 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 28 28 28 54 54 54 T T T 6 120-139 27 27 27 50 50 50 T T T 7 140-159 29 29 29 55 55 55 T T T 8 160-179 26 26 26 46 46 46 T T T 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 24 24 24 42 42 42 T T T 12 240-259 30 30 30 56 56 56 T T T 13 260-279 24 24 24 42 42 42 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 28 28 28 51 51 51 F F F 18 360-379 28 28 28 51 51 51 F F F 19 380-399 29 29 29 55 55 55 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 30 30 30 56 56 56 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 30 30 30 56 56 56 F F F 24 480-499 30 30 30 56 56 56 F F F 25 500-519 28 28 28 54 54 54 F F F 26 520-539 30 30 30 56 56 56 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 28 28 28 52 52 52 F F F 29 580-599 30 30 30 56 56 56 F F F Max gen: 30, Max flush: 56 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 56 56 56 F F F 1 020-039 30 30 30 56 56 56 F F F 2 040-059 29 29 29 55 55 55 F F F 3 060-079 27 27 27 50 50 50 F F F 4 080-099 21 21 21 39 39 39 F F F 5 100-119 31 31 31 57 57 57 F F F 6 120-139 31 31 31 57 57 57 F F F 7 140-159 31 31 31 57 57 57 F F F 8 160-179 31 31 31 57 57 57 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 25 25 25 44 44 44 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 31 31 31 57 57 57 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 26 26 26 49 49 49 F F F 17 340-359 28 28 28 51 51 51 F F F 18 360-379 28 28 28 51 51 51 F F F 19 380-399 29 29 29 55 55 55 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 30 30 30 56 56 56 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 30 30 30 56 56 56 F F F 24 480-499 30 30 30 56 56 56 F F F 25 500-519 28 28 28 54 54 54 F F F 26 520-539 30 30 30 56 56 56 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 28 28 28 52 52 52 F F F 29 580-599 30 30 30 56 56 56 F F F Max gen: 31, Max flush: 57 Begin loop 16 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.240853304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241126803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241143091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311} {"msg":"Crucible 8111d754-17be-4e35-bce4-345f3a55b78a has session id: 96f6b29c-1d53-44fd-a442-a5092df31e60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241515605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Upstairs opts: Upstairs UUID: 8111d754-17be-4e35-bce4-345f3a55b78a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241563928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Crucible stats registered with UUID: 8111d754-17be-4e35-bce4-345f3a55b78a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241583305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241902768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311} {"msg":"8111d754-17be-4e35-bce4-345f3a55b78a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241940252Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241953761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241971018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.241995464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.24202112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242044067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242067564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.24209155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242115017Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242140133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_connection connected from Ok(127.0.0.1:64652)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242315227Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_connection connected from Ok(127.0.0.1:58352)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242349782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:03.242573049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:03.242611083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:03.242704829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242722556Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242736094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.242848678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.243251538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.243272105Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.243286633Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.243300271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.243927948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.243968222Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.244497923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.244535808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.244564483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.244596569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.244625385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:03.244644642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.24465588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_connection connected from Ok(127.0.0.1:33895)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.244870729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"io task","client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.245775424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.24580365Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246355428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.24688705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246909886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[0]R flush_numbers[0..12]: [56, 56, 55, 50, 39, 57, 57, 57, 57, 44, 44, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246925184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[0]R generation[0..12]: [30, 30, 29, 27, 21, 31, 31, 31, 31, 25, 25, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246939492Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.24695389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[1]R flush_numbers[0..12]: [56, 56, 55, 50, 39, 57, 57, 57, 57, 44, 44, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246967458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[1]R generation[0..12]: [30, 30, 29, 27, 21, 31, 31, 31, 31, 25, 25, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246981225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.246999483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[2]R flush_numbers[0..12]: [56, 56, 55, 50, 39, 57, 57, 57, 57, 44, 44, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247014181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[2]R generation[0..12]: [30, 30, 29, 27, 21, 31, 31, 31, 31, 25, 25, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247028898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247052305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:15:13.247074852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247097558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247120485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247134273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"":"downstairs","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247147341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247162429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"0","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247185295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"1","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247209802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"client":"2","":"downstairs","session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247231528Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247252895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"8111d754-17be-4e35-bce4-345f3a55b78a is now active with session: 96f6b29c-1d53-44fd-a442-a5092df31e60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247277282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247304418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311,"session_id":"96f6b29c-1d53-44fd-a442-a5092df31e60"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:13.247320055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 210 len 12288 data: 2 2 3 02/30 Flush 03/30 Write block 83 len 36864 data: 4 4 3 3 3 3 3 3 2 04/30 Read block 71 len 20480 05/30 Read block 99 len 4096 06/30 Write block 325 len 20480 data: 3 3 2 2 2 07/30 Read block 400 len 36864 08/30 Read block 170 len 16384 09/30 Write block 503 len 20480 data: 2 4 4 4 5 10/30 Read block 312 len 24576 11/30 Write block 570 len 16384 data: 4 6 4 4 12/30 Flush 13/30 Flush 14/30 Flush 15/30 Write block 585 len 36864 data: 2 3 3 3 3 3 3 3 2 16/30 Read block 293 len 28672 17/30 Read block 237 len 16384 18/30 Flush 19/30 Write block 323 len 32768 data: 3 3 4 4 3 3 3 2 20/30 Flush 21/30 Write block 359 len 36864 data: 6 7 6 4 3 3 3 3 4 22/30 Write block 389 len 4096 data: 5 23/30 Write block 373 len 16384 data: 6 6 5 3 24/30 Read block 248 len 36864 25/30 Read block 549 len 24576 26/30 Read block 511 len 16384 27/30 Write block 331 len 16384 data: 2 2 3 3 28/30 Read block 58 len 36864 29/30 Read block 49 len 16384 30/30 Write block 522 len 28672 data: 4 3 3 2 2 2 2 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 9 Done Done Done false 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:13.274756848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1311} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 56 56 56 F F F 1 020-039 30 30 30 56 56 56 F F F 2 040-059 29 29 29 55 55 55 F F F 3 060-079 27 27 27 50 50 50 F F F 4 080-099 32 32 32 59 59 59 F F F 5 100-119 31 31 31 57 57 57 F F F 6 120-139 31 31 31 57 57 57 F F F 7 140-159 31 31 31 57 57 57 F F F 8 160-179 31 31 31 57 57 57 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 31 31 31 57 57 57 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 32 32 32 64 64 64 T T T 17 340-359 28 28 28 51 51 51 T T T 18 360-379 28 28 28 51 51 51 T T T 19 380-399 29 29 29 55 55 55 T T T 20 400-419 29 29 29 55 55 55 F F F 21 420-439 30 30 30 56 56 56 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 30 30 30 56 56 56 F F F 24 480-499 30 30 30 56 56 56 F F F 25 500-519 32 32 32 59 59 59 F F F 26 520-539 30 30 30 56 56 56 T T T 27 540-559 26 26 26 49 49 49 F F F 28 560-579 32 32 32 59 59 59 F F F 29 580-599 32 32 32 62 62 62 F F F Max gen: 32, Max flush: 64 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 56 56 56 F F F 1 020-039 30 30 30 56 56 56 F F F 2 040-059 29 29 29 55 55 55 F F F 3 060-079 27 27 27 50 50 50 F F F 4 080-099 32 32 32 59 59 59 F F F 5 100-119 31 31 31 57 57 57 F F F 6 120-139 31 31 31 57 57 57 F F F 7 140-159 31 31 31 57 57 57 F F F 8 160-179 31 31 31 57 57 57 F F F 9 180-199 25 25 25 44 44 44 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 31 31 31 57 57 57 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 33 33 33 65 65 65 F F F 17 340-359 33 33 33 65 65 65 F F F 18 360-379 33 33 33 65 65 65 F F F 19 380-399 33 33 33 65 65 65 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 30 30 30 56 56 56 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 30 30 30 56 56 56 F F F 24 480-499 30 30 30 56 56 56 F F F 25 500-519 32 32 32 59 59 59 F F F 26 520-539 33 33 33 65 65 65 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 32 32 32 59 59 59 F F F 29 580-599 32 32 32 62 62 62 F F F Max gen: 33, Max flush: 65 Begin loop 17 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.493598705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.49403764Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494069155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319} {"msg":"Crucible b27b40a7-4155-4ee6-b96f-60410ea15ca0 has session id: fe7bd46c-baaa-4c97-9c8c-b3d423369e31","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494231531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Upstairs opts: Upstairs UUID: b27b40a7-4155-4ee6-b96f-60410ea15ca0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494282874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Crucible stats registered with UUID: b27b40a7-4155-4ee6-b96f-60410ea15ca0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.49430605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.49464597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319} {"msg":"b27b40a7-4155-4ee6-b96f-60410ea15ca0 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494744295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494761913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494776281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494789829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494803097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494816065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494829433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.49484504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494858938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.494875716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:27.495074617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:53050)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495113011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:27.495146866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:27.49525374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495286395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495313581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:60966)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495337718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495360574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495788361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495814447Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495829125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.495842513Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.496368705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.496495646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.496958757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.496978054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.496992182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.49707499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.497093657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:27.497106885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.495867539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_connection connected from Ok(127.0.0.1:47195)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.495924711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"io task","client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.496904445Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.496956118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.497509996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498039557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498067213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[0]R flush_numbers[0..12]: [56, 56, 55, 50, 59, 57, 57, 57, 57, 44, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.49808404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[0]R generation[0..12]: [30, 30, 29, 27, 32, 31, 31, 31, 31, 25, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498098768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498113266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[1]R flush_numbers[0..12]: [56, 56, 55, 50, 59, 57, 57, 57, 57, 44, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498127004Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[1]R generation[0..12]: [30, 30, 29, 27, 32, 31, 31, 31, 31, 25, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498140562Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498159249Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[2]R flush_numbers[0..12]: [56, 56, 55, 50, 59, 57, 57, 57, 57, 44, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498173137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[2]R generation[0..12]: [30, 30, 29, 27, 32, 31, 31, 31, 31, 25, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498190515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498204283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498216961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498230269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498243057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498255905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"":"downstairs","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498269023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498280751Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"0","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498293659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"1","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498306667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"client":"2","":"downstairs","session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498319745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498331904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"b27b40a7-4155-4ee6-b96f-60410ea15ca0 is now active with session: fe7bd46c-baaa-4c97-9c8c-b3d423369e31","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498344762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.49835866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319,"session_id":"fe7bd46c-baaa-4c97-9c8c-b3d423369e31"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:37.498370968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 36864 data: 4 4 4 6 6 6 6 8 8 02/30 Write block 103 len 40960 data: 3 4 4 4 4 5 5 4 5 6 03/30 Write block 186 len 40960 data: 3 3 3 3 3 3 2 2 3 4 04/30 Read block 369 len 16384 05/30 Read block 149 len 40960 06/30 Read block 202 len 16384 07/30 Write block 22 len 32768 data: 4 4 4 3 3 4 4 4 08/30 Read block 497 len 32768 09/30 Read block 268 len 4096 10/30 Flush 11/30 Write block 69 len 4096 data: 3 12/30 Read block 566 len 20480 13/30 Write block 40 len 24576 data: 2 2 3 3 4 3 14/30 Write block 560 len 36864 data: 3 3 4 6 5 5 3 3 3 15/30 Read block 64 len 20480 16/30 Write block 336 len 4096 data: 3 17/30 Read block 135 len 36864 18/30 Read block 144 len 16384 19/30 Read block 186 len 12288 20/30 Write block 487 len 4096 data: 7 21/30 Write block 335 len 24576 data: 3 4 4 4 4 3 22/30 Write block 517 len 32768 data: 6 6 6 6 6 5 4 4 23/30 Read block 290 len 12288 24/30 Write block 88 len 40960 data: 4 4 4 3 2 2 2 2 2 2 25/30 Write block 375 len 4096 data: 6 26/30 Read block 327 len 32768 27/30 Write block 473 len 24576 data: 3 3 4 5 4 4 28/30 Write block 127 len 16384 data: 3 2 3 3 29/30 Read block 157 len 36864 30/30 Write block 593 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Write 10 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:37.524272179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1319} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 56 56 56 F F F 1 020-039 34 34 34 66 66 66 F F F 2 040-059 34 34 34 67 67 67 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 32 32 32 59 59 59 T T T 5 100-119 34 34 34 66 66 66 F F F 6 120-139 31 31 31 57 57 57 T T T 7 140-159 31 31 31 57 57 57 F F F 8 160-179 31 31 31 57 57 57 F F F 9 180-199 34 34 34 66 66 66 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 31 31 31 57 57 57 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 34 34 34 67 67 67 T T T 17 340-359 33 33 33 65 65 65 T T T 18 360-379 33 33 33 65 65 65 T T T 19 380-399 33 33 33 65 65 65 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 30 30 30 56 56 56 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 30 30 30 56 56 56 T T T 24 480-499 34 34 34 67 67 67 F F F 25 500-519 32 32 32 59 59 59 T T T 26 520-539 33 33 33 65 65 65 T T T 27 540-559 26 26 26 49 49 49 F F F 28 560-579 34 34 34 67 67 67 F F F 29 580-599 32 32 32 62 62 62 T T T Max gen: 34, Max flush: 67 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 56 56 56 F F F 1 020-039 34 34 34 66 66 66 F F F 2 040-059 34 34 34 67 67 67 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 68 68 68 F F F 5 100-119 34 34 34 66 66 66 F F F 6 120-139 35 35 35 68 68 68 F F F 7 140-159 31 31 31 57 57 57 F F F 8 160-179 31 31 31 57 57 57 F F F 9 180-199 34 34 34 66 66 66 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 31 31 31 57 57 57 F F F 13 260-279 24 24 24 42 42 42 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 F F F 16 320-339 35 35 35 68 68 68 F F F 17 340-359 35 35 35 68 68 68 F F F 18 360-379 35 35 35 68 68 68 F F F 19 380-399 33 33 33 65 65 65 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 30 30 30 56 56 56 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 35 35 35 68 68 68 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 35 35 35 68 68 68 F F F 26 520-539 35 35 35 68 68 68 F F F 27 540-559 26 26 26 49 49 49 F F F 28 560-579 34 34 34 67 67 67 F F F 29 580-599 35 35 35 68 68 68 F F F Max gen: 35, Max flush: 68 Begin loop 18 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.786933051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.787359498Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.787388913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327} {"msg":"Crucible 217af11b-d5c2-4162-be12-b1669319f555 has session id: b023b91d-b874-48ce-9a50-e8b8f66cc60f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.787556869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Upstairs opts: Upstairs UUID: 217af11b-d5c2-4162-be12-b1669319f555, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.787607151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Crucible stats registered with UUID: 217af11b-d5c2-4162-be12-b1669319f555","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.787631897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788005042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327} {"msg":"217af11b-d5c2-4162-be12-b1669319f555 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788106367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788132203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.78815358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788178907Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788205383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788231489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788257205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788282661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788308677Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788334403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45348)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.78836116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:51.788392435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:51.78842619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:15:51.788454476Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788475423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788553611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48774)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788585906Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.788615902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.789171059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.789197376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.789211603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.7892337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.789710819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.789819553Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.790162333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.790193318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.790212445Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.790225093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.790238501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:15:51.790251429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.789630121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44723)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.789753073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"io task","client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.790873757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.790914641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.791428585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.79200404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792032005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[0]R flush_numbers[0..12]: [56, 66, 67, 67, 68, 66, 68, 57, 57, 66, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792048743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[0]R generation[0..12]: [30, 34, 34, 34, 35, 34, 35, 31, 31, 34, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792063581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792077929Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[1]R flush_numbers[0..12]: [56, 66, 67, 67, 68, 66, 68, 57, 57, 66, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792092196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[1]R generation[0..12]: [30, 34, 34, 34, 35, 34, 35, 31, 31, 34, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792110524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792124892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[2]R flush_numbers[0..12]: [56, 66, 67, 67, 68, 66, 68, 57, 57, 66, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.79213858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[2]R generation[0..12]: [30, 34, 34, 34, 35, 34, 35, 31, 31, 34, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792152078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792166125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792178884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792195531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792208459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792221507Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"":"downstairs","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792234706Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792246624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"0","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792259622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"1","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.79227263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"client":"2","":"downstairs","session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792285648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792297986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"217af11b-d5c2-4162-be12-b1669319f555 is now active with session: b023b91d-b874-48ce-9a50-e8b8f66cc60f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792313144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.792325322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327,"session_id":"b023b91d-b874-48ce-9a50-e8b8f66cc60f"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:01.79233741Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 110 len 16384 data: 5 6 7 7 02/30 Write block 430 len 4096 data: 5 03/30 Write block 257 len 20480 data: 6 6 5 4 5 04/30 Read block 186 len 4096 05/30 Read block 47 len 12288 06/30 Read block 325 len 32768 07/30 Write block 542 len 40960 data: 3 3 3 3 4 4 3 3 2 2 08/30 Write block 42 len 24576 data: 4 4 5 4 2 2 09/30 Write block 578 len 20480 data: 4 4 5 5 3 10/30 Write block 1 len 40960 data: 3 3 3 2 2 2 2 3 4 4 11/30 Write block 323 len 40960 data: 4 4 5 5 4 4 4 3 3 3 12/30 Write block 425 len 12288 data: 3 2 4 13/30 Read block 61 len 32768 14/30 Read block 188 len 20480 15/30 Read block 220 len 36864 16/30 Read block 242 len 16384 17/30 Flush 18/30 Write block 590 len 32768 data: 4 4 4 4 4 4 4 3 19/30 Read block 182 len 20480 20/30 Read block 264 len 36864 21/30 Write block 316 len 8192 data: 4 4 22/30 Read block 172 len 4096 23/30 Read block 184 len 16384 24/30 Write block 175 len 36864 data: 4 4 5 4 4 4 3 2 3 25/30 Write block 390 len 20480 data: 5 5 4 3 5 26/30 Read block 395 len 36864 27/30 Read block 156 len 16384 28/30 Write block 158 len 4096 data: 6 29/30 Write block 568 len 20480 data: 4 3 5 7 5 30/30 Write block 467 len 4096 data: 5 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Read 5 Done Done Done false 20 Acked 1019 Read 9 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 5 Sent Sent Sent false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:01.816394174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1327} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 34 34 34 66 66 66 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 68 68 68 F F F 5 100-119 36 36 36 69 69 69 F F F 6 120-139 35 35 35 68 68 68 F F F 7 140-159 31 31 31 57 57 57 T T T 8 160-179 31 31 31 57 57 57 T T T 9 180-199 34 34 34 66 66 66 T T T 10 200-219 32 32 32 58 58 58 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 36 36 36 69 69 69 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 20 20 20 37 37 37 T T T 16 320-339 36 36 36 69 69 69 F F F 17 340-359 35 35 35 68 68 68 F F F 18 360-379 35 35 35 68 68 68 F F F 19 380-399 33 33 33 65 65 65 T T T 20 400-419 29 29 29 55 55 55 F F F 21 420-439 36 36 36 69 69 69 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 35 35 35 68 68 68 T T T 24 480-499 34 34 34 67 67 67 F F F 25 500-519 35 35 35 68 68 68 F F F 26 520-539 35 35 35 68 68 68 F F F 27 540-559 36 36 36 69 69 69 F F F 28 560-579 36 36 36 69 69 69 T T T 29 580-599 36 36 36 69 69 69 T T T Max gen: 36, Max flush: 69 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 34 34 34 66 66 66 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 68 68 68 F F F 5 100-119 36 36 36 69 69 69 F F F 6 120-139 35 35 35 68 68 68 F F F 7 140-159 37 37 37 70 70 70 F F F 8 160-179 37 37 37 70 70 70 F F F 9 180-199 37 37 37 70 70 70 F F F 10 200-219 32 32 32 58 58 58 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 36 36 36 69 69 69 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 30 30 30 56 56 56 F F F 15 300-319 37 37 37 70 70 70 F F F 16 320-339 36 36 36 69 69 69 F F F 17 340-359 35 35 35 68 68 68 F F F 18 360-379 35 35 35 68 68 68 F F F 19 380-399 37 37 37 70 70 70 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 36 36 36 69 69 69 F F F 22 440-459 26 26 26 46 46 46 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 35 35 35 68 68 68 F F F 26 520-539 35 35 35 68 68 68 F F F 27 540-559 36 36 36 69 69 69 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 37, Max flush: 70 Begin loop 19 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.070885559Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.071263292Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.071299337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335} {"msg":"Crucible 85ce13a2-5663-445d-b641-c8e688b56e84 has session id: 19f45785-372a-49b2-b87d-8d54b2882a2a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.071563128Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Upstairs opts: Upstairs UUID: 85ce13a2-5663-445d-b641-c8e688b56e84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.071608391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Crucible stats registered with UUID: 85ce13a2-5663-445d-b641-c8e688b56e84","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.071631348Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072005853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335} {"msg":"85ce13a2-5663-445d-b641-c8e688b56e84 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072042697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072058825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072073862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.07208962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072113826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072140382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072165989Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072188905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072214012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072237358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:16.072354991Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:49052)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072389466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:16.072416322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:61072)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072441128Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:16.072469014Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.07249297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072516917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.072710178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.073183258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.073210144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.073225962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.07324003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.073822733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.073861927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.074416375Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.074461058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.074482455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.074505392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.074523399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:16.074537097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.07403663Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56496)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.074094862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"io task","client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.075257609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.075287245Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.075882357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076517563Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076545838Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[0]R flush_numbers[0..12]: [69, 66, 69, 67, 68, 69, 68, 70, 70, 70, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076562546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[0]R generation[0..12]: [36, 34, 36, 34, 35, 36, 35, 37, 37, 37, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076577724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076592081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[1]R flush_numbers[0..12]: [69, 66, 69, 67, 68, 69, 68, 70, 70, 70, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076610559Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[1]R generation[0..12]: [36, 34, 36, 34, 35, 36, 35, 37, 37, 37, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076624637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076638585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[2]R flush_numbers[0..12]: [69, 66, 69, 67, 68, 69, 68, 70, 70, 70, 58, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076651953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[2]R generation[0..12]: [36, 34, 36, 34, 35, 36, 35, 37, 37, 37, 32, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076665551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076679339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:16:26.076692097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076707744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076720722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076748668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"":"downstairs","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076762586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076774474Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"0","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076787443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"1","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076800501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"client":"2","":"downstairs","session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076813589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076828916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"85ce13a2-5663-445d-b641-c8e688b56e84 is now active with session: 19f45785-372a-49b2-b87d-8d54b2882a2a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076842024Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076853743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335,"session_id":"19f45785-372a-49b2-b87d-8d54b2882a2a"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:26.076865861Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 8192 data: 6 5 02/30 Read block 206 len 12288 03/30 Write block 286 len 8192 data: 5 7 04/30 Read block 119 len 40960 05/30 Write block 31 len 12288 data: 3 3 4 06/30 Write block 523 len 12288 data: 5 5 3 07/30 Write block 379 len 16384 data: 4 5 4 4 08/30 Write block 436 len 24576 data: 4 4 4 3 3 4 09/30 Write block 199 len 16384 data: 5 5 5 5 10/30 Read block 12 len 16384 11/30 Write block 114 len 12288 data: 6 7 6 12/30 Read block 44 len 24576 13/30 Flush 14/30 Flush 15/30 Write block 144 len 16384 data: 4 4 3 3 16/30 Write block 553 len 16384 data: 2 3 3 3 17/30 Read block 596 len 8192 18/30 Write block 553 len 4096 data: 3 19/30 Read block 444 len 24576 20/30 Flush 21/30 Write block 373 len 32768 data: 7 7 7 4 3 4 5 6 22/30 Write block 165 len 12288 data: 6 5 5 23/30 Flush 24/30 Write block 141 len 20480 data: 3 3 4 5 5 25/30 Write block 343 len 20480 data: 4 4 5 5 3 26/30 Read block 586 len 8192 27/30 Write block 361 len 32768 data: 7 5 4 4 4 4 5 5 28/30 Read block 571 len 20480 29/30 Read block 266 len 16384 30/30 Read block 89 len 36864 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 7 6 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:26.10331605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1335} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 38 38 38 71 71 71 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 68 68 68 F F F 5 100-119 38 38 38 71 71 71 F F F 6 120-139 35 35 35 68 68 68 F F F 7 140-159 38 38 38 73 73 73 T T T 8 160-179 38 38 38 74 74 74 F F F 9 180-199 38 38 38 71 71 71 F F F 10 200-219 38 38 38 71 71 71 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 36 36 36 69 69 69 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 37 37 37 70 70 70 F F F 16 320-339 36 36 36 69 69 69 F F F 17 340-359 35 35 35 68 68 68 T T T 18 360-379 38 38 38 74 74 74 T T T 19 380-399 38 38 38 74 74 74 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 38 38 38 71 71 71 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 35 35 35 68 68 68 F F F 26 520-539 38 38 38 71 71 71 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 38, Max flush: 74 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 38 38 38 71 71 71 F F F 2 040-059 36 36 36 69 69 69 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 68 68 68 F F F 5 100-119 38 38 38 71 71 71 F F F 6 120-139 35 35 35 68 68 68 F F F 7 140-159 39 39 39 75 75 75 F F F 8 160-179 38 38 38 74 74 74 F F F 9 180-199 38 38 38 71 71 71 F F F 10 200-219 38 38 38 71 71 71 F F F 11 220-239 31 31 31 57 57 57 F F F 12 240-259 36 36 36 69 69 69 F F F 13 260-279 36 36 36 69 69 69 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 37 37 37 70 70 70 F F F 16 320-339 36 36 36 69 69 69 F F F 17 340-359 39 39 39 75 75 75 F F F 18 360-379 39 39 39 75 75 75 F F F 19 380-399 38 38 38 74 74 74 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 38 38 38 71 71 71 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 37 37 37 70 70 70 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 35 35 35 68 68 68 F F F 26 520-539 38 38 38 71 71 71 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 39, Max flush: 75 Begin loop 20 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.293132991Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.293846045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.29387848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343} {"msg":"Crucible 2d90b421-ede9-412c-8028-e51263703337 has session id: 02938889-b5b2-4fb8-8c8d-9c4942f68ea1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.293906786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Upstairs opts: Upstairs UUID: 2d90b421-ede9-412c-8028-e51263703337, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.293941611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Crucible stats registered with UUID: 2d90b421-ede9-412c-8028-e51263703337","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.293963977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.2942836Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343} {"msg":"2d90b421-ede9-412c-8028-e51263703337 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294390474Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294414721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294442796Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294467523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294491879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294517075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294539842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294563099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294588135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294611681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_connection connected from Ok(127.0.0.1:45880)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.294848996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62380)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.29489395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:40.295020631Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:40.295056635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:40.295083701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.295104248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.295125175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.295222001Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.295578528Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.295604524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.295619552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.29563303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296330897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296364241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296881425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296909591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296923769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296941666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296955424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:40.296968742Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.296559403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_connection connected from Ok(127.0.0.1:55708)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.296733367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"io task","client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.297650241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.297677887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.298338789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.298919053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.298942019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[0]R flush_numbers[0..12]: [69, 71, 69, 67, 68, 71, 68, 75, 74, 71, 71, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.298957447Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[0]R generation[0..12]: [36, 38, 36, 34, 35, 38, 35, 39, 38, 38, 38, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.298972225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.298986373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[1]R flush_numbers[0..12]: [69, 71, 69, 67, 68, 71, 68, 75, 74, 71, 71, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299000031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[1]R generation[0..12]: [36, 38, 36, 34, 35, 38, 35, 39, 38, 38, 38, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299083829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299103495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[2]R flush_numbers[0..12]: [69, 71, 69, 67, 68, 71, 68, 75, 74, 71, 71, 57]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299117373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[2]R generation[0..12]: [36, 38, 36, 34, 35, 38, 35, 39, 38, 38, 38, 31]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299130731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299144629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299157347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299170496Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Next flush: 76","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299183384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299196252Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"":"downstairs","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.29920974Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299222068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"0","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299235356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"1","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level"Wait for a query_work_queue command to finish before sending IO :30,"time":"2024-08-05T19:16:50.299251793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"client":"2","":"downstairs","session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299265142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299280229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"2d90b421-ede9-412c-8028-e51263703337 is now active with session: 02938889-b5b2-4fb8-8c8d-9c4942f68ea1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299293147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299305016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343,"session_id":"02938889-b5b2-4fb8-8c8d-9c4942f68ea1"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:16:50.299317224Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 92 len 8192 data: 3 3 02/30 Read block 80 len 4096 03/30 Write block 462 len 24576 data: 4 3 3 3 3 6 04/30 Write block 321 len 8192 data: 3 3 05/30 Write block 161 len 36864 data: 7 8 8 8 7 6 6 4 4 06/30 Read block 485 len 20480 07/30 Flush 08/30 Write block 245 len 36864 data: 3 3 2 2 4 5 6 7 7 09/30 Write block 46 len 8192 data: 3 3 10/30 Read block 286 len 12288 11/30 Write block 105 len 8192 data: 5 5 12/30 Read block 93 len 24576 13/30 Write block 220 len 24576 data: 5 4 2 2 2 2 14/30 Write block 271 len 24576 data: 4 4 4 5 5 4 15/30 Write block 44 len 24576 data: 6 5 4 4 2 3 16/30 Write block 173 len 4096 data: 4 17/30 Write block 466 len 16384 data: 4 7 3 3 18/30 Flush 19/30 Write block 508 len 20480 data: 5 5 4 4 4 20/30 Write block 313 len 12288 data: 4 4 4 21/30 Read block 9 len 16384 22/30 Read block 137 len 24576 23/30 Read block 515 len 12288 24/30 Read block 541 len 24576 25/30 Flush 26/30 Write block 210 len 8192 data: 3 3 27/30 Read block 259 len 40960 28/30 Read block 307 len 36864 29/30 Read block 551 len 12288 30/30 Write block 127 len 36864 data: 4 3 4 4 3 3 3 4 4 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false 31 Acked 1030 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:16:50.326165784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1343} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 38 38 38 71 71 71 F F F 2 040-059 40 40 40 77 77 77 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 40 40 40 76 76 76 F F F 5 100-119 40 40 40 77 77 77 F F F 6 120-139 35 35 35 68 68 68 T T T 7 140-159 39 39 39 75 75 75 F F F 8 160-179 40 40 40 77 77 77 F F F 9 180-199 38 38 38 71 71 71 F F F 10 200-219 38 38 38 71 71 71 T T T 11 220-239 40 40 40 77 77 77 F F F 12 240-259 40 40 40 77 77 77 F F F 13 260-279 40 40 40 77 77 77 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 40 40 40 79 79 79 F F F 16 320-339 40 40 40 76 76 76 F F F 17 340-359 39 39 39 75 75 75 F F F 18 360-379 39 39 39 75 75 75 F F F 19 380-399 38 38 38 74 74 74 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 38 38 38 71 71 71 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 38 38 38 71 71 71 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 40, Max flush: 79 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 38 38 38 71 71 71 F F F 2 040-059 40 40 40 77 77 77 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 40 40 40 76 76 76 F F F 5 100-119 40 40 40 77 77 77 F F F 6 120-139 41 41 41 80 80 80 F F F 7 140-159 39 39 39 75 75 75 F F F 8 160-179 40 40 40 77 77 77 F F F 9 180-199 38 38 38 71 71 71 F F F 10 200-219 41 41 41 80 80 80 F F F 11 220-239 40 40 40 77 77 77 F F F 12 240-259 40 40 40 77 77 77 F F F 13 260-279 40 40 40 77 77 77 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 40 40 40 79 79 79 F F F 16 320-339 40 40 40 76 76 76 F F F 17 340-359 39 39 39 75 75 75 F F F 18 360-379 39 39 39 75 75 75 F F F 19 380-399 38 38 38 74 74 74 F F F 20 400-419 29 29 29 55 55 55 F F F 21 420-439 38 38 38 71 71 71 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 38 38 38 71 71 71 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 41, Max flush: 80 Begin loop 21 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.509106864Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.509590882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.509621808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351} {"msg":"Crucible a4a6111e-873e-4339-be23-eb05102ce3ea has session id: 78518d07-3242-4fef-ae00-141cbed52259","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.509792632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Upstairs opts: Upstairs UUID: a4a6111e-873e-4339-be23-eb05102ce3ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.509830117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Crucible stats registered with UUID: a4a6111e-873e-4339-be23-eb05102ce3ea","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.509845805Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510219739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351} {"msg":"a4a6111e-873e-4339-be23-eb05102ce3ea active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510315185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510340961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510358058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510382065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510409161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510435847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510459853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.51048043Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510503117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510531423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_connection connected from Ok(127.0.0.1:61986)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510756819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_connection connected from Ok(127.0.0.1:42506)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.510796733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:04.511010272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:04.511049586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:04.511137043Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.51115658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.511173058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.511212052Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.511598575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.511623111Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.511741273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.511759231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512097771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512213743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512593847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512630532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512658437Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512737106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512756973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:04.512770941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.51256739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_connection connected from Ok(127.0.0.1:34789)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.512675804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"io task","client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.51364764Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.513675246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514196179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514733999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514760605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[0]R flush_numbers[0..12]: [69, 71, 77, 67, 76, 77, 80, 75, 77, 71, 80, 77]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514776462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[0]R generation[0..12]: [36, 38, 40, 34, 40, 40, 41, 39, 40, 38, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.51479104Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514805558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[1]R flush_numbers[0..12]: [69, 71, 77, 67, 76, 77, 80, 75, 77, 71, 80, 77]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514819146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[1]R generation[0..12]: [36, 38, 40, 34, 40, 40, 41, 39, 40, 38, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514837413Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514851731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[2]R flush_numbers[0..12]: [69, 71, 77, 67, 76, 77, 80, 75, 77, 71, 80, 77]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514865489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[2]R generation[0..12]: [36, 38, 40, 34, 40, 40, 41, 39, 40, 38, 41, 40]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514879087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514892835Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514909493Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514922931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514935679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514948517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"":"downstairs","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514961865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514973933Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"0","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.514987381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"1","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.515000629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"client":"2","":"downstairs","session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.515014097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.515026405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"a4a6111e-873e-4339-be23-eb05102ce3ea is now active with session: 78518d07-3242-4fef-ae00-141cbed52259","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.515041173Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.515053101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351,"session_id":"78518d07-3242-4fef-ae00-141cbed52259"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:14.51506522Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 72 len 20480 data: 2 2 2 2 2 02/30 Write block 377 len 20480 data: 4 5 6 7 5 03/30 Read block 376 len 32768 04/30 Write block 328 len 16384 data: 5 5 4 4 05/30 Read block 224 len 12288 06/30 Read block 470 len 4096 07/30 Write block 364 len 36864 data: 5 5 5 6 6 5 5 5 5 08/30 Write block 525 len 40960 data: 4 3 3 3 2 2 3 4 4 4 09/30 Write block 423 len 20480 data: 3 3 4 3 5 10/30 Write block 226 len 16384 data: 2 2 2 3 11/30 Write block 253 len 8192 data: 8 6 12/30 Write block 262 len 16384 data: 5 4 4 5 13/30 Read block 449 len 24576 14/30 Read block 73 len 40960 15/30 Flush 16/30 Write block 55 len 28672 data: 5 5 5 4 4 4 3 17/30 Write block 211 len 40960 data: 4 4 4 4 4 4 5 4 5 6 18/30 Write block 244 len 32768 data: 3 4 4 3 3 5 6 7 19/30 Write block 82 len 4096 data: 4 20/30 Write block 395 len 28672 data: 5 6 4 4 3 4 5 21/30 Read block 289 len 40960 22/30 Write block 186 len 28672 data: 4 4 4 4 4 4 3 23/30 Write block 27 len 32768 data: 5 5 5 3 4 4 5 3 24/30 Read block 318 len 32768 25/30 Read block 6 len 28672 26/30 Read block 533 len 40960 27/30 Read block 203 len 28672 28/30 Write block 527 len 4096 data: 4 29/30 Read block 74 len 24576 30/30 Write block 111 len 40960 data: 7 8 8 7 8 7 5 6 5 5 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 7 Done Done Done false 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Write 1 Done Done Done false 20 Acked 1019 Write 7 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:14.540206773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1351} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 38 38 38 71 71 71 T T T 2 040-059 40 40 40 77 77 77 T T T 3 060-079 42 42 42 81 81 81 T T T 4 080-099 40 40 40 76 76 76 T T T 5 100-119 40 40 40 77 77 77 T T T 6 120-139 41 41 41 80 80 80 T T T 7 140-159 39 39 39 75 75 75 F F F 8 160-179 40 40 40 77 77 77 F F F 9 180-199 38 38 38 71 71 71 T T T 10 200-219 41 41 41 80 80 80 T T T 11 220-239 42 42 42 81 81 81 T T T 12 240-259 42 42 42 81 81 81 T T T 13 260-279 42 42 42 81 81 81 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 40 40 40 79 79 79 F F F 16 320-339 42 42 42 81 81 81 F F F 17 340-359 39 39 39 75 75 75 F F F 18 360-379 42 42 42 81 81 81 F F F 19 380-399 42 42 42 81 81 81 T T T 20 400-419 29 29 29 55 55 55 T T T 21 420-439 42 42 42 81 81 81 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 42 42 42 81 81 81 T T T 27 540-559 38 38 38 73 73 73 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 42, Max flush: 81 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 36 36 36 69 69 69 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 43 43 43 82 82 82 F F F 5 100-119 43 43 43 82 82 82 F F F 6 120-139 43 43 43 82 82 82 F F F 7 140-159 39 39 39 75 75 75 F F F 8 160-179 40 40 40 77 77 77 F F F 9 180-199 43 43 43 82 82 82 F F F 10 200-219 43 43 43 82 82 82 F F F 11 220-239 43 43 43 82 82 82 F F F 12 240-259 43 43 43 82 82 82 F F F 13 260-279 42 42 42 81 81 81 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 40 40 40 79 79 79 F F F 16 320-339 42 42 42 81 81 81 F F F 17 340-359 39 39 39 75 75 75 F F F 18 360-379 42 42 42 81 81 81 F F F 19 380-399 43 43 43 82 82 82 F F F 20 400-419 43 43 43 82 82 82 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 34 34 34 67 67 67 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 43 43 43 82 82 82 F F F 27 540-559 38 38 38 73 73 73 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 F F F Max gen: 43, Max flush: 82 Begin loop 22 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.855158614Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.855730939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.855763134Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359} {"msg":"Crucible f2002433-d3ee-41ee-a39c-9b723890cc9b has session id: d2f8325e-795f-4a24-8e65-730c7fe1b50d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.855868138Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Upstairs opts: Upstairs UUID: f2002433-d3ee-41ee-a39c-9b723890cc9b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.855912212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Crucible stats registered with UUID: f2002433-d3ee-41ee-a39c-9b723890cc9b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.855935599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856220416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359} {"msg":"f2002433-d3ee-41ee-a39c-9b723890cc9b active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856317112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856341338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856367635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856393471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856416707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856441193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.85646627Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856489846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856514233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856540379Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:28.856641964Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:28.856677069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:28.856704085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856723282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_connection connected from Ok(127.0.0.1:62002)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856747298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856772104Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856846093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59948)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.856932851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.857365676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.857393203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.85740808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.857421618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.858041116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.85808159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.858586455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.858611772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.85862574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.858688051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.858707058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:28.858720716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.858328893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39914)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.858540341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"io task","client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.859454986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.859482762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860031161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860620753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860656758Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[0]R flush_numbers[0..12]: [69, 82, 82, 82, 82, 82, 82, 75, 77, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860673345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[0]R generation[0..12]: [36, 43, 43, 43, 43, 43, 43, 39, 40, 43, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860687873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.86070694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[1]R flush_numbers[0..12]: [69, 82, 82, 82, 82, 82, 82, 75, 77, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860721288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[1]R generation[0..12]: [36, 43, 43, 43, 43, 43, 43, 39, 40, 43, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860734946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860748694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[2]R flush_numbers[0..12]: [69, 82, 82, 82, 82, 82, 82, 75, 77, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860762262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[2]R generation[0..12]: [36, 43, 43, 43, 43, 43, 43, 39, 40, 43, 43, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.86077567Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860789498Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860805636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860818874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Next flush: 83","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860831442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.86084401Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"":"downstairs","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860857268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860869216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"0","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860882345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"1","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860895272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"client":"2","":"downstairs","session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.86091052Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860922659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"f2002433-d3ee-41ee-a39c-9b723890cc9b is now active with session: d2f8325e-795f-4a24-8e65-730c7fe1b50d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860935266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860946945Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359,"session_id":"d2f8325e-795f-4a24-8e65-730c7fe1b50d"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:38.860958903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 334 len 40960 data: 4 4 5 5 5 5 4 3 4 5 02/30 Read block 417 len 32768 03/30 Write block 169 len 16384 data: 5 3 3 3 04/30 Read block 219 len 16384 05/30 Read block 151 len 36864 06/30 Flush 07/30 Write block 7 len 28672 data: 3 4 5 5 4 4 3 08/30 Read block 505 len 36864 09/30 Write block 554 len 4096 data: 4 10/30 Flush 11/30 Write block 162 len 16384 data: 9 9 9 8 12/30 Read block 384 len 8192 13/30 Write block 315 len 24576 data: 5 5 5 3 2 3 14/30 Read block 184 len 32768 15/30 Write block 326 len 32768 data: 6 5 6 6 5 5 4 4 16/30 Read block 116 len 24576 17/30 Write block 595 len 4096 data: 5 18/30 Write block 488 len 20480 data: 9 9 8 6 4 19/30 Read block 524 len 16384 20/30 Read block 360 len 4096 21/30 Read block 524 len 28672 22/30 Write block 92 len 16384 data: 4 4 3 3 23/30 Write block 391 len 40960 data: 6 5 4 6 6 7 5 5 4 5 24/30 Write block 140 len 28672 data: 3 4 4 5 6 6 4 25/30 Read block 450 len 4096 26/30 Write block 492 len 24576 data: 5 3 3 3 3 4 27/30 Write block 206 len 8192 data: 4 4 28/30 Read block 305 len 40960 29/30 Write block 400 len 28672 data: 6 6 6 6 7 7 7 30/30 Write block 171 len 16384 data: 4 4 5 4 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Read 6 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Read 1 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Write 4 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:38.887077142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1359} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 84 84 84 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 43 43 43 82 82 82 T T T 5 100-119 43 43 43 82 82 82 F F F 6 120-139 43 43 43 82 82 82 F F F 7 140-159 39 39 39 75 75 75 T T T 8 160-179 44 44 44 85 85 85 T T T 9 180-199 43 43 43 82 82 82 F F F 10 200-219 43 43 43 82 82 82 T T T 11 220-239 43 43 43 82 82 82 F F F 12 240-259 43 43 43 82 82 82 F F F 13 260-279 42 42 42 81 81 81 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 44 44 44 85 85 85 F F F 16 320-339 44 44 44 85 85 85 T T T 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 81 81 81 F F F 19 380-399 43 43 43 82 82 82 T T T 20 400-419 43 43 43 82 82 82 T T T 21 420-439 42 42 42 81 81 81 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 34 34 34 67 67 67 T T T 25 500-519 40 40 40 78 78 78 F F F 26 520-539 43 43 43 82 82 82 F F F 27 540-559 44 44 44 84 84 84 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 37 37 37 70 70 70 T T T Max gen: 44, Max flush: 85 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 44 44 44 84 84 84 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 45 45 45 86 86 86 F F F 5 100-119 43 43 43 82 82 82 F F F 6 120-139 43 43 43 82 82 82 F F F 7 140-159 45 45 45 86 86 86 F F F 8 160-179 45 45 45 86 86 86 F F F 9 180-199 43 43 43 82 82 82 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 43 43 43 82 82 82 F F F 12 240-259 43 43 43 82 82 82 F F F 13 260-279 42 42 42 81 81 81 F F F 14 280-299 38 38 38 71 71 71 F F F 15 300-319 44 44 44 85 85 85 F F F 16 320-339 45 45 45 86 86 86 F F F 17 340-359 44 44 44 83 83 83 F F F 18 360-379 42 42 42 81 81 81 F F F 19 380-399 45 45 45 86 86 86 F F F 20 400-419 45 45 45 86 86 86 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 38 38 38 71 71 71 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 43 43 43 82 82 82 F F F 27 540-559 44 44 44 84 84 84 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 45 45 45 86 86 86 F F F Max gen: 45, Max flush: 86 Begin loop 23 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.158762975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159155957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159186733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367} {"msg":"Crucible fa0c117f-0cf6-4041-aa97-c7b7e0db0a7d has session id: 78b17748-25b2-41b7-b307-eee5685f97ed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159446344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Upstairs opts: Upstairs UUID: fa0c117f-0cf6-4041-aa97-c7b7e0db0a7d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159580824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Crucible stats registered with UUID: fa0c117f-0cf6-4041-aa97-c7b7e0db0a7d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159616679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159939831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367} {"msg":"fa0c117f-0cf6-4041-aa97-c7b7e0db0a7d active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159962418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159976196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.159991113Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160004762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160018369Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160031398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160053844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160076211Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160094748Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160108716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:48035)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160530973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:64596)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160572177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:53.160599423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:53.160616191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:17:53.160632078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160644847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160658115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.160747531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.16116214Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.161188806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.161203704Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.161217322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.161808164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.16183565Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.162367851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.162394297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.162408425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.162420823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.162434471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:17:53.16244748Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.162105719Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:63802)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.162213403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"io task","client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.163151854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.163185819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.163792239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164382612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164412927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[0]R flush_numbers[0..12]: [84, 82, 82, 82, 86, 82, 82, 86, 86, 82, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164427685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[0]R generation[0..12]: [44, 43, 43, 43, 45, 43, 43, 45, 45, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164446842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.16446172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[1]R flush_numbers[0..12]: [84, 82, 82, 82, 86, 82, 82, 86, 86, 82, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164475898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[1]R generation[0..12]: [44, 43, 43, 43, 45, 43, 43, 45, 45, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164489396Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164503524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[2]R flush_numbers[0..12]: [84, 82, 82, 82, 86, 82, 82, 86, 86, 82, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164517312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[2]R generation[0..12]: [44, 43, 43, 43, 45, 43, 43, 45, 45, 43, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.16453165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164545468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164558386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164571554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164587911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164601519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"":"downstairs","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164614847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164626916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"0","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164642533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"1","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164655801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"client":"2","":"downstairs","session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164669039Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164681177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"fa0c117f-0cf6-4041-aa97-c7b7e0db0a7d is now active with session: 78b17748-25b2-41b7-b307-eee5685f97ed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164694015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164705924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367,"session_id":"78b17748-25b2-41b7-b307-eee5685f97ed"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:03.164718052Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 252 len 12288 data: 8 9 7 02/30 Read block 6 len 40960 03/30 Flush 04/30 Write block 15 len 4096 data: 4 05/30 Read block 568 len 20480 06/30 Write block 360 len 40960 data: 8 8 6 5 6 6 6 7 7 6 07/30 Read block 414 len 4096 08/30 Read block 156 len 8192 09/30 Write block 170 len 8192 data: 4 5 10/30 Flush 11/30 Flush 12/30 Write block 296 len 12288 data: 5 5 3 13/30 Read block 527 len 4096 14/30 Read block 591 len 4096 15/30 Write block 597 len 4096 data: 4 16/30 Read block 488 len 36864 17/30 Read block 452 len 20480 18/30 Read block 529 len 12288 19/30 Read block 318 len 36864 20/30 Write block 446 len 36864 data: 4 3 4 4 5 4 4 5 5 21/30 Flush 22/30 Write block 386 len 20480 data: 4 4 5 6 6 23/30 Read block 463 len 12288 24/30 Read block 467 len 12288 25/30 Read block 100 len 24576 26/30 Write block 309 len 8192 data: 2 3 27/30 Write block 529 len 12288 data: 3 3 4 28/30 Write block 174 len 36864 data: 5 5 5 6 5 5 5 4 3 29/30 Read block 573 len 40960 30/30 Read block 427 len 24576 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Write 3 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:03.189316786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1367} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 88 88 88 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 45 45 45 86 86 86 F F F 5 100-119 43 43 43 82 82 82 F F F 6 120-139 43 43 43 82 82 82 F F F 7 140-159 45 45 45 86 86 86 F F F 8 160-179 46 46 46 88 88 88 T T T 9 180-199 43 43 43 82 82 82 T T T 10 200-219 45 45 45 86 86 86 F F F 11 220-239 43 43 43 82 82 82 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 42 42 42 81 81 81 F F F 14 280-299 46 46 46 90 90 90 F F F 15 300-319 44 44 44 85 85 85 T T T 16 320-339 45 45 45 86 86 86 F F F 17 340-359 44 44 44 83 83 83 F F F 18 360-379 46 46 46 88 88 88 F F F 19 380-399 45 45 45 86 86 86 T T T 20 400-419 45 45 45 86 86 86 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 46 46 46 90 90 90 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 43 43 43 82 82 82 T T T 27 540-559 44 44 44 84 84 84 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 46 46 46 90 90 90 F F F Max gen: 46, Max flush: 90 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 88 88 88 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 45 45 45 86 86 86 F F F 5 100-119 43 43 43 82 82 82 F F F 6 120-139 43 43 43 82 82 82 F F F 7 140-159 45 45 45 86 86 86 F F F 8 160-179 47 47 47 91 91 91 F F F 9 180-199 47 47 47 91 91 91 F F F 10 200-219 45 45 45 86 86 86 F F F 11 220-239 43 43 43 82 82 82 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 42 42 42 81 81 81 F F F 14 280-299 46 46 46 90 90 90 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 45 45 45 86 86 86 F F F 17 340-359 44 44 44 83 83 83 F F F 18 360-379 46 46 46 88 88 88 F F F 19 380-399 47 47 47 91 91 91 F F F 20 400-419 45 45 45 86 86 86 F F F 21 420-439 42 42 42 81 81 81 F F F 22 440-459 46 46 46 90 90 90 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 47 47 47 91 91 91 F F F 27 540-559 44 44 44 84 84 84 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 46 46 46 90 90 90 F F F Max gen: 47, Max flush: 91 Begin loop 24 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.407618623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408169951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408203116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375} {"msg":"Crucible 6bdedd9b-af95-4a32-b024-b0054304d5a0 has session id: 9cc9f0fb-2d1d-4e7e-b93d-9dcada308063","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.40824318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Upstairs opts: Upstairs UUID: 6bdedd9b-af95-4a32-b024-b0054304d5a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408277285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Crucible stats registered with UUID: 6bdedd9b-af95-4a32-b024-b0054304d5a0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408302422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408634552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375} {"msg":"6bdedd9b-af95-4a32-b024-b0054304d5a0 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408731688Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408750115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408764893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.40878151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408805787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408831073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408857739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408879136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408897184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.408918Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_connection connected from Ok(127.0.0.1:39902)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409062739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_connection connected from Ok(127.0.0.1:34468)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409150106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:17.409303623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:17.409340728Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:17.409377632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409399059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409421236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409565844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409831665Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.409870159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.41000284Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410040214Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410326112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410440335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410871621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410894457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410908435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410926083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410939661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:17.410952539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.410693847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_connection connected from Ok(127.0.0.1:64774)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.410817199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"io task","client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.411714126Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.41174945Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412301708Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.41283488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412862635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[0]R flush_numbers[0..12]: [88, 82, 82, 82, 86, 82, 82, 86, 91, 91, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412879443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[0]R generation[0..12]: [46, 43, 43, 43, 45, 43, 43, 45, 47, 47, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.41289927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412914008Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[1]R flush_numbers[0..12]: [88, 82, 82, 82, 86, 82, 82, 86, 91, 91, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412927706Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[1]R generation[0..12]: [46, 43, 43, 43, 45, 43, 43, 45, 47, 47, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412941314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412955671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[2]R flush_numbers[0..12]: [88, 82, 82, 82, 86, 82, 82, 86, 91, 91, 86, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412969509Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[2]R generation[0..12]: [46, 43, 43, 43, 45, 43, 43, 45, 47, 47, 45, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412983207Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.412997295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Max found gen is 48","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2024-08-05T19:18:27.413013853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413027221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Next flush: 92","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413039779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413052807Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"":"downstairs","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413066005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413077973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"0","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413091722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"1","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413106939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"client":"2","":"downstairs","session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413120407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413132575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"6bdedd9b-af95-4a32-b024-b0054304d5a0 is now active with session: 9cc9f0fb-2d1d-4e7e-b93d-9dcada308063","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413145753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.413157472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375,"session_id":"9cc9f0fb-2d1d-4e7e-b93d-9dcada308063"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:27.41316951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 338 len 20480 data: 6 6 5 4 5 02/30 Write block 181 len 20480 data: 5 4 4 3 3 03/30 Flush 04/30 Write block 407 len 4096 data: 7 05/30 Flush 06/30 Write block 268 len 12288 data: 4 4 3 07/30 Read block 286 len 24576 08/30 Write block 553 len 12288 data: 4 5 4 09/30 Read block 410 len 8192 10/30 Write block 106 len 32768 data: 6 5 6 6 6 8 9 9 11/30 Read block 79 len 24576 12/30 Read block 178 len 32768 13/30 Read block 156 len 36864 14/30 Write block 448 len 24576 data: 5 5 6 5 5 6 15/30 Read block 352 len 16384 16/30 Write block 423 len 4096 data: 4 17/30 Write block 207 len 40960 data: 5 3 2 4 5 5 5 5 5 5 18/30 Read block 476 len 20480 19/30 Read block 294 len 36864 20/30 Write block 120 len 8192 data: 6 5 21/30 Write block 533 len 12288 data: 5 5 4 22/30 Write block 373 len 36864 data: 8 8 8 5 5 6 7 8 6 23/30 Write block 355 len 4096 data: 4 24/30 Write block 440 len 12288 data: 4 5 6 25/30 Flush 26/30 Flush 27/30 Write block 85 len 20480 data: 4 4 4 5 5 28/30 Write block 365 len 12288 data: 7 7 8 29/30 Write block 173 len 4096 data: 6 30/30 Read block 303 len 36864 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Write 1 Done Done Sent false 31 Acked 1030 Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 2 3 Done 3 4 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:27.440077901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1375} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 88 88 88 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 45 45 45 86 86 86 T T T 5 100-119 48 48 48 94 94 94 F F F 6 120-139 48 48 48 95 95 95 F F F 7 140-159 45 45 45 86 86 86 F F F 8 160-179 47 47 47 91 91 91 T T T 9 180-199 48 48 48 92 92 92 F F F 10 200-219 48 48 48 94 94 94 F F F 11 220-239 43 43 43 82 82 82 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 46 46 46 90 90 90 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 48 48 48 92 92 92 F F F 17 340-359 48 48 48 95 95 95 F F F 18 360-379 48 48 48 95 95 95 T T T 19 380-399 48 48 48 95 95 95 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 48 48 48 95 95 95 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 46 46 46 90 90 90 F F F Max gen: 48, Max flush: 95 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 88 88 88 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 49 49 49 96 96 96 F F F 5 100-119 48 48 48 94 94 94 F F F 6 120-139 48 48 48 95 95 95 F F F 7 140-159 45 45 45 86 86 86 F F F 8 160-179 49 49 49 96 96 96 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 48 48 48 94 94 94 F F F 11 220-239 43 43 43 82 82 82 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 46 46 46 90 90 90 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 48 48 48 92 92 92 F F F 17 340-359 48 48 48 95 95 95 F F F 18 360-379 49 49 49 96 96 96 F F F 19 380-399 48 48 48 95 95 95 F F F 20 400-419 48 48 48 93 93 93 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 48 48 48 95 95 95 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 40 40 40 78 78 78 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 37 37 37 70 70 70 F F F 29 580-599 46 46 46 90 90 90 F F F Max gen: 49, Max flush: 96 Begin loop 25 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.631173172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.631638133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.631673058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383} {"msg":"Crucible 713004f8-2c40-406e-afdc-9488a6288813 has session id: 9298c9ac-85f0-43e4-9bd3-bd91f101e505","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.631718371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Upstairs opts: Upstairs UUID: 713004f8-2c40-406e-afdc-9488a6288813, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.631756586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Crucible stats registered with UUID: 713004f8-2c40-406e-afdc-9488a6288813","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.631783931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632095056Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383} {"msg":"713004f8-2c40-406e-afdc-9488a6288813 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632125851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632141558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632165175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632191861Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632217167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632239984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.63226314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632288227Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632314503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632344879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:41.632393831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:41.632422697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:41.632443734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.63246585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632488767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632621797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_connection connected from Ok(127.0.0.1:59878)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632661272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_connection connected from Ok(127.0.0.1:56501)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.632688348Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.633201861Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.633228428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.633333162Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.633390953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.633732233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.633841957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.634307317Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.634347172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.634377627Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.634400324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.634415541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:41.634429139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.634203483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_connection connected from Ok(127.0.0.1:42951)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.63442785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"io task","client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.635295302Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.635328826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.63584517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636388409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636421975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[0]R flush_numbers[0..12]: [88, 82, 82, 82, 96, 94, 95, 86, 96, 92, 94, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636437992Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[0]R generation[0..12]: [46, 43, 43, 43, 49, 48, 48, 45, 49, 48, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.63645265Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636471967Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[1]R flush_numbers[0..12]: [88, 82, 82, 82, 96, 94, 95, 86, 96, 92, 94, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636485925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[1]R generation[0..12]: [46, 43, 43, 43, 49, 48, 48, 45, 49, 48, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636499393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636513571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[2]R flush_numbers[0..12]: [88, 82, 82, 82, 96, 94, 95, 86, 96, 92, 94, 82]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636526879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[2]R generation[0..12]: [46, 43, 43, 43, 49, 48, 48, 45, 49, 48, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636540287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636554135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636566813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Generation requested: 50 >= found:50",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.63658344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636596319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636609047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"":"downstairs","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636622085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636633893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"0","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636646831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"1","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636659669Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"client":"2","":"downstairs","session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636674827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636687135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"713004f8-2c40-406e-afdc-9488a6288813 is now active with session: 9298c9ac-85f0-43e4-9bd3-bd91f101e505","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636699793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.636711592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383,"session_id":"9298c9ac-85f0-43e4-9bd3-bd91f101e505"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:18:51.63672354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 565 len 12288 data: 6 4 4 02/30 Read block 505 len 16384 03/30 Read block 265 len 16384 04/30 Write block 336 len 8192 data: 6 6 05/30 Read block 349 len 16384 06/30 Write block 561 len 4096 data: 4 07/30 Write block 112 len 20480 data: 10 10 8 9 8 08/30 Write block 577 len 16384 data: 3 5 5 6 09/30 Read block 509 len 8192 10/30 Flush 11/30 Write block 9 len 36864 data: 6 6 5 5 4 3 5 4 3 12/30 Read block 554 len 20480 13/30 Flush 14/30 Write block 405 len 4096 data: 8 15/30 Read block 40 len 36864 16/30 Flush 17/30 Write block 504 len 12288 data: 5 5 5 18/30 Write block 156 len 8192 data: 8 8 19/30 Read block 569 len 20480 20/30 Read block 335 len 28672 21/30 Write block 232 len 28672 data: 3 3 3 3 4 3 4 22/30 Read block 354 len 36864 23/30 Read block 326 len 4096 24/30 Read block 309 len 8192 25/30 Read block 245 len 20480 26/30 Read block 432 len 36864 27/30 Write block 168 len 32768 data: 5 6 5 6 5 7 6 6 28/30 Read block 507 len 24576 29/30 Read block 539 len 36864 30/30 Read block 86 len 32768 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 7 Done Done Done false 23 Acked 1022 Read 9 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:18:51.661335432Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1383} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 98 98 98 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 49 49 49 96 96 96 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 48 48 48 95 95 95 F F F 7 140-159 50 50 50 100 100 100 F F F 8 160-179 49 49 49 96 96 96 T T T 9 180-199 48 48 48 92 92 92 F F F 10 200-219 48 48 48 94 94 94 F F F 11 220-239 43 43 43 82 82 82 T T T 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 46 46 46 90 90 90 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 50 50 50 97 97 97 F F F 17 340-359 48 48 48 95 95 95 F F F 18 360-379 49 49 49 96 96 96 F F F 19 380-399 48 48 48 95 95 95 F F F 20 400-419 50 50 50 99 99 99 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 48 48 48 95 95 95 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 50 50 50 100 100 100 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 50 50 50 97 97 97 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 50, Max flush: 100 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 98 98 98 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 43 43 43 82 82 82 F F F 3 060-079 43 43 43 82 82 82 F F F 4 080-099 49 49 49 96 96 96 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 48 48 48 95 95 95 F F F 7 140-159 50 50 50 100 100 100 F F F 8 160-179 51 51 51 101 101 101 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 48 48 48 94 94 94 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 46 46 46 90 90 90 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 50 50 50 97 97 97 F F F 17 340-359 48 48 48 95 95 95 F F F 18 360-379 49 49 49 96 96 96 F F F 19 380-399 48 48 48 95 95 95 F F F 20 400-419 50 50 50 99 99 99 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 48 48 48 95 95 95 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 50 50 50 100 100 100 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 48 48 48 94 94 94 F F F 28 560-579 50 50 50 97 97 97 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 51, Max flush: 101 Begin loop 26 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.839130583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.839800464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.839831949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391} {"msg":"Crucible a97aaca8-e659-4cc4-9283-c608170751ae has session id: 352dd67b-e757-4d75-83b6-988754b06e64","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.839846937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Upstairs opts: Upstairs UUID: a97aaca8-e659-4cc4-9283-c608170751ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.839868474Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Crucible stats registered with UUID: a97aaca8-e659-4cc4-9283-c608170751ae","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.839882262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840149832Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391} {"msg":"a97aaca8-e659-4cc4-9283-c608170751ae active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.84023392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840252117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840267285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840280883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840294521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840307649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840320877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840419932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840455247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840480503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:34875)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840631171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:48376)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840785968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:05.840836171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:05.840870785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:05.8409711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.840989598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.841003706Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.84110827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.841520269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.841572971Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.841603767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.841631282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842204988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842240632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842686546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842711533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.84272569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842787041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842805328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:05.842818617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.842270007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_connection connected from Ok(127.0.0.1:47376)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.842372402Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"io task","client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.843350047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.843378162Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.84393538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844492517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844520433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[0]R flush_numbers[0..12]: [98, 82, 82, 82, 96, 97, 95, 100, 101, 92, 94, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844537051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[0]R generation[0..12]: [50, 43, 43, 43, 49, 50, 48, 50, 51, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844551428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844570346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[1]R flush_numbers[0..12]: [98, 82, 82, 82, 96, 97, 95, 100, 101, 92, 94, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844584524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[1]R generation[0..12]: [50, 43, 43, 43, 49, 50, 48, 50, 51, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844598232Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844612159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[2]R flush_numbers[0..12]: [98, 82, 82, 82, 96, 97, 95, 100, 101, 92, 94, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844626077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[2]R generation[0..12]: [50, 43, 43, 43, 49, 50, 48, 50, 51, 48, 48, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844639735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844653604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844669721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844682969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Next flush: 102","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844696097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844709305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"":"downstairs","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844722473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844736931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"0","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844759928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"1","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844785244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"client":"2","":"downstairs","session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844816029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844841425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"a97aaca8-e659-4cc4-9283-c608170751ae is now active with session: 352dd67b-e757-4d75-83b6-988754b06e64","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844865192Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844887399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391,"session_id":"352dd67b-e757-4d75-83b6-988754b06e64"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:15.844909775Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 283 len 36864 data: 5 5 4 6 8 6 5 7 6 02/30 Read block 120 len 32768 03/30 Read block 358 len 12288 04/30 Write block 59 len 16384 data: 5 5 4 3 05/30 Write block 352 len 20480 data: 5 4 4 5 3 06/30 Write block 341 len 40960 data: 5 6 6 5 6 6 4 5 5 5 07/30 Write block 207 len 20480 data: 6 4 3 5 6 08/30 Read block 462 len 20480 09/30 Write block 206 len 32768 data: 5 7 5 4 6 7 6 6 10/30 Write block 411 len 12288 data: 6 5 4 11/30 Read block 191 len 28672 12/30 Read block 495 len 16384 13/30 Read block 537 len 36864 14/30 Read block 59 len 40960 15/30 Write block 387 len 16384 data: 5 6 7 7 16/30 Read block 425 len 28672 17/30 Read block 553 len 8192 18/30 Write block 137 len 40960 data: 4 3 3 4 5 5 6 7 7 5 19/30 Flush 20/30 Write block 504 len 32768 data: 6 6 6 6 6 6 5 5 21/30 Read block 238 len 32768 22/30 Read block 316 len 16384 23/30 Read block 521 len 40960 24/30 Write block 160 len 4096 data: 8 25/30 Write block 556 len 24576 data: 4 2 3 3 4 5 26/30 Write block 324 len 20480 data: 5 6 7 6 7 27/30 Read block 241 len 16384 28/30 Write block 504 len 36864 data: 7 7 7 7 7 7 6 6 5 29/30 Read block 58 len 24576 30/30 Read block 392 len 12288 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 8 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 5 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:15.87078423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1391} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 98 98 98 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 52 52 52 102 102 102 F F F 3 060-079 52 52 52 102 102 102 F F F 4 080-099 49 49 49 96 96 96 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 51 51 51 101 101 101 T T T 9 180-199 48 48 48 92 92 92 F F F 10 200-219 52 52 52 102 102 102 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 52 52 52 102 102 102 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 50 50 50 97 97 97 T T T 17 340-359 52 52 52 102 102 102 F F F 18 360-379 49 49 49 96 96 96 F F F 19 380-399 52 52 52 102 102 102 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 48 48 48 95 95 95 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 50 50 50 100 100 100 T T T 26 520-539 48 48 48 95 95 95 F F F 27 540-559 48 48 48 94 94 94 T T T 28 560-579 50 50 50 97 97 97 T T T 29 580-599 50 50 50 97 97 97 F F F Max gen: 52, Max flush: 102 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 98 98 98 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 52 52 52 102 102 102 F F F 3 060-079 52 52 52 102 102 102 F F F 4 080-099 49 49 49 96 96 96 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 53 53 53 103 103 103 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 52 52 52 102 102 102 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 52 52 52 102 102 102 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 49 49 49 96 96 96 F F F 19 380-399 52 52 52 102 102 102 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 48 48 48 95 95 95 F F F 23 460-479 40 40 40 77 77 77 F F F 24 480-499 45 45 45 86 86 86 F F F 25 500-519 53 53 53 103 103 103 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 53 53 53 103 103 103 F F F 28 560-579 53 53 53 103 103 103 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 53, Max flush: 103 Begin loop 27 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.086212564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.086678545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.086707571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399} {"msg":"Crucible 476bfddd-e950-43d5-860a-3db47129f2c4 has session id: b16e72c3-eac6-43af-8abc-b0910b2cdcd6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.086888274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Upstairs opts: Upstairs UUID: 476bfddd-e950-43d5-860a-3db47129f2c4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.086929998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Crucible stats registered with UUID: 476bfddd-e950-43d5-860a-3db47129f2c4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.086953185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087351495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399} {"msg":"476bfddd-e950-43d5-860a-3db47129f2c4 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087403488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087420195Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087435353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087449101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087462839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087476097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087495534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087515291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087531219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087548516Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:30.087736168Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:30.087774593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35436)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087801189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:30.087828855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087852551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087877777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:59259)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087959075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.087985681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.088632156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.08867099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.088701635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.088730221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.0892101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089328002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089670802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089694438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089708586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089845976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089863963Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:30.089877161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.089364136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_connection connected from Ok(127.0.0.1:56540)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.089553978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"io task","client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.090431158Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.090457244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091073203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091680703Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.09170359Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[0]R flush_numbers[0..12]: [98, 82, 102, 102, 96, 97, 102, 102, 103, 92, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091728656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[0]R generation[0..12]: [50, 43, 52, 52, 49, 50, 52, 52, 53, 48, 52, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091743354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091762771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[1]R flush_numbers[0..12]: [98, 82, 102, 102, 96, 97, 102, 102, 103, 92, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091777199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[1]R generation[0..12]: [50, 43, 52, 52, 49, 50, 52, 52, 53, 48, 52, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091790877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091804725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[2]R flush_numbers[0..12]: [98, 82, 102, 102, 96, 97, 102, 102, 103, 92, 102, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091818383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[2]R generation[0..12]: [50, 43, 52, 52, 49, 50, 52, 52, 53, 48, 52, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091831821Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091845719Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091858577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:19:40.091871505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091887732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.09190078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"":"downstairs","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091913878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091925687Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"0","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091938895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"1","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091954862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"client":"2","":"downstairs","session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.09196849Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091980659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"476bfddd-e950-43d5-860a-3db47129f2c4 is now active with session: b16e72c3-eac6-43af-8abc-b0910b2cdcd6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.091993487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.092020943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399,"session_id":"b16e72c3-eac6-43af-8abc-b0910b2cdcd6"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:40.092033381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 60 len 20480 data: 6 5 4 3 3 02/30 Write block 561 len 32768 data: 6 5 7 6 7 5 5 5 03/30 Write block 89 len 8192 data: 6 5 04/30 Write block 505 len 20480 data: 8 8 8 8 8 05/30 Write block 165 len 32768 data: 9 7 7 6 7 6 7 6 06/30 Write block 52 len 40960 data: 4 4 4 6 6 6 5 6 7 6 07/30 Flush 08/30 Write block 441 len 36864 data: 6 7 6 5 5 5 4 6 6 09/30 Read block 503 len 16384 10/30 Write block 94 len 20480 data: 4 4 3 3 2 11/30 Read block 153 len 8192 12/30 Read block 321 len 12288 13/30 Write block 550 len 32768 data: 3 3 2 5 6 5 5 3 14/30 Read block 396 len 32768 15/30 Write block 93 len 16384 data: 5 5 5 4 16/30 Write block 378 len 28672 data: 7 8 9 7 5 4 4 17/30 Flush 18/30 Write block 209 len 16384 data: 5 7 8 7 19/30 Read block 238 len 32768 20/30 Write block 280 len 4096 data: 4 21/30 Read block 296 len 24576 22/30 Read block 92 len 20480 23/30 Read block 301 len 24576 24/30 Write block 165 len 20480 data: 10 8 8 7 8 25/30 Flush 26/30 Write block 69 len 24576 data: 4 3 2 3 3 3 27/30 Write block 289 len 16384 data: 6 8 7 6 28/30 Write block 477 len 36864 data: 5 5 5 5 5 5 5 7 7 29/30 Read block 520 len 8192 30/30 Read block 42 len 32768 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:40.119660506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1399} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 98 98 98 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 54 54 54 104 104 104 T T T 4 080-099 54 54 54 105 105 105 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 54 54 54 106 106 106 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 54 54 54 106 106 106 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 54 54 54 106 106 106 T T T 15 300-319 47 47 47 91 91 91 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 54 54 54 105 105 105 F F F 19 380-399 54 54 54 105 105 105 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 40 40 40 77 77 77 T T T 24 480-499 45 45 45 86 86 86 T T T 25 500-519 54 54 54 104 104 104 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 54, Max flush: 106 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 98 98 98 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 55 55 55 107 107 107 F F F 4 080-099 54 54 54 105 105 105 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 52 52 52 102 102 102 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 54 54 54 106 106 106 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 54 54 54 106 106 106 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 48 48 48 94 94 94 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 47 47 47 91 91 91 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 54 54 54 105 105 105 F F F 19 380-399 54 54 54 105 105 105 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 55 55 55 107 107 107 F F F 24 480-499 55 55 55 107 107 107 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 48 48 48 95 95 95 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 55, Max flush: 107 Begin loop 28 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.326696824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327273588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327316622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407} {"msg":"Crucible b613f098-168c-41fb-98ef-a86c00cae8ed has session id: 624eee8b-2e17-42fc-9d7f-7cb123975cb5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327506294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Upstairs opts: Upstairs UUID: b613f098-168c-41fb-98ef-a86c00cae8ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327540329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Crucible stats registered with UUID: b613f098-168c-41fb-98ef-a86c00cae8ed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327555246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327916593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407} {"msg":"b613f098-168c-41fb-98ef-a86c00cae8ed active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.327983193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328016608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328035805Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328050473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.32806769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328091617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328109234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328122532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.32813621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328162756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:54.328282739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42562)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328318523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:36223)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328346469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:54.328377755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:19:54.32841066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328436356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328461592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328538451Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.32894844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.328975786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.329089849Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.329127044Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.329531233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.329666913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.330018461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.330041218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.330055786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.330138403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.330156791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:19:54.330170179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.329601853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_connection connected from Ok(127.0.0.1:44492)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.329677182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"io task","client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.330746793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.330781998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.331349374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.331933467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.331961403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[0]R flush_numbers[0..12]: [98, 82, 104, 107, 105, 97, 102, 102, 106, 92, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.331978151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[0]R generation[0..12]: [50, 43, 54, 55, 54, 50, 52, 52, 54, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.331992539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332011916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[1]R flush_numbers[0..12]: [98, 82, 104, 107, 105, 97, 102, 102, 106, 92, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332026154Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[1]R generation[0..12]: [50, 43, 54, 55, 54, 50, 52, 52, 54, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332039882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.33205403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[2]R flush_numbers[0..12]: [98, 82, 104, 107, 105, 97, 102, 102, 106, 92, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332067768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[2]R generation[0..12]: [50, 43, 54, 55, 54, 50, 52, 52, 54, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332081215Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332095203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332108082Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"}Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332125419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332138387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332151335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"":"downstairs","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332164613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332176901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"0","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332190049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"1","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332205197Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"client":"2","":"downstairs","session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332218765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332231023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"b613f098-168c-41fb-98ef-a86c00cae8ed is now active with session: 624eee8b-2e17-42fc-9d7f-7cb123975cb5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332243761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.33225564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:04.332267798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 462 len 8192 data: 5 4 04/30 Read block 531 len 12288 05/30 Write block 77 len 32768 data: 3 5 5 5 4 5 5 5 06/30 Read block 13 len 12288 07/30 Read block 112 len 36864 08/30 Write block 269 len 36864 data: 5 4 5 5 5 6 6 5 4 09/30 Write block 296 len 20480 data: 6 6 4 5 4 10/30 Write block 488 len 28672 data: 10 10 9 7 6 4 4 11/30 Read block 351 len 8192 12/30 Write block 158 len 40960 data: 7 6 9 8 10 10 10 11 9 9 13/30 Read block 315 len 4096 14/30 Read block 402 len 12288 15/30 Write block 311 len 20480 data: 4 4 5 5 6 16/30 Read block 316 len 40960 17/30 Read block 182 len 20480 18/30 Write block 19 len 4096 data: 3 19/30 Read block 204 len 4096 20/30 Read block 338 len 8192 21/30 Write block 312 len 32768 data: 5 6 6 7 6 6 4 3 22/30 Write block 523 len 8192 data: 6 6 23/30 Read block 244 len 40960 24/30 Write block 131 len 36864 data: 4 4 4 5 5 4 5 4 4 25/30 Write block 364 len 16384 data: 7 8 8 9 26/30 Write block 64 len 16384 data: 4 4 3 3 27/30 Read block 525 len 4096 28/30 Read block 335 len 16384 29/30 Read block 195 len 28672 30/30 Read block 334 len 8192 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 3 Acked 1002 Write 2 Done Done Done false 4 Acked 1003 Read 3 Done Done Done false 5 Acked 1004 Write 8 Done Done Done false 6 Acked 1005 Read 3 Done Done Done false 7 Acked 1006 Read 9 Done Done Done false 8 Acked 1007 Write 9 Done Done Done false 9 Acked 1008 Write 5 Done Done Done false 10 Acked 1009 Write 7 Done Done Done false 11 Acked 1010 Read 2 Done Done Done false 12 Acked 1011 Write 10 Done Done Done false 13 Acked 1012 Read 1 Done Done Done false 14 Acked 1013 Read 3 Done Done Done false 15 Acked 1014 Write 5 Done Done Done false 16 Acked 1015 Read 10 Done Done Done false 17 Acked 1016 Read 5 Done Done Done false 18 Acked 1017 Write 1 Done Done Done false 19 Acked 1018 Read 1 Done Done Done false 20 Acked 1019 Read 2 Done Done Done false 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Flush 0 Done Done Sent false 29 Acked 1028 Read 4 Done Done Sent false 30 Acked 1029 Read 7 Done Done Sent false 31 Acked 1030 Read 2 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 4 4 Done 29 29 25 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1027 1027 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 31 28 30 29 27 Wrote out file "/tmp/test_repair_verify.data" {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358056355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358100949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35816172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358191965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358218631Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358391895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358425081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358447747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358504889Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358529675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358549042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35856196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358576848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358589286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358601484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358613613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358625581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358637619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358649567Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358661486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358673424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358685372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35869731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358709359Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358721707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358733825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358746113Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358758331Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35877049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358784417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358796526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358808454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358820422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35883319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358845048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358856837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358868595Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358880393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358892301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35890428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358916438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358928676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358940594Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358952603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358964551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358976599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.358988527Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359009134Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359027711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359040339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359052368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359064336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359076214Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359088042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359099851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359111869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359123777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359135715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359147654Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359159482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35917146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359183738Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359195886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359208125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359222032Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359234301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359246459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359258577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359270705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359282834Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359294952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35930709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359319118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359330996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359342745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359354623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359366481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359378419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359390347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359402366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359414424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359427852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35943986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359452029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359463887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359475895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359487843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.3595112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359524558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359536976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359549064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359561212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35957322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359585299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359597067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359609035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359621003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359632752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359646729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359658938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359670836Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359682854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359694802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359706881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359718919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359730887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359742765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359754793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359766592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.35977864Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359790608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359802337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359813925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359825873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:04.359837851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1407,"session_id":"624eee8b-2e17-42fc-9d7f-7cb123975cb5"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 110 110 110 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 F F F 4 080-099 56 56 56 110 110 110 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 56 56 56 110 110 110 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 54 54 54 106 106 106 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 56 56 56 110 110 110 F F F 14 280-299 56 56 56 110 110 110 F F F 15 300-319 56 56 56 110 110 110 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 56 56 56 110 110 110 F F F 19 380-399 54 54 54 105 105 105 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 56 56 56 110 110 110 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 56, Max flush: 110 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 56 56 56 110 110 110 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 F F F 4 080-099 56 56 56 110 110 110 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 56 56 56 110 110 110 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 54 54 54 106 106 106 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 56 56 56 110 110 110 F F F 14 280-299 56 56 56 110 110 110 F F F 15 300-319 56 56 56 110 110 110 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 52 52 52 102 102 102 F F F 18 360-379 56 56 56 110 110 110 F F F 19 380-399 54 54 54 105 105 105 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 56 56 56 110 110 110 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 56, Max flush: 110 Begin loop 29 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.515044256Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.515672903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.515706588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415} {"msg":"Crucible f29be442-cdc0-479a-b8a6-f7f1b2f05d7a has session id: 96faf315-62e1-49a5-9608-6367454cff9d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.515728404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Upstairs opts: Upstairs UUID: f29be442-cdc0-479a-b8a6-f7f1b2f05d7a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.515760989Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Crucible stats registered with UUID: f29be442-cdc0-479a-b8a6-f7f1b2f05d7a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.515778267Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516071163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415} {"msg":"f29be442-cdc0-479a-b8a6-f7f1b2f05d7a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516185327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516202524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516216962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.51623223Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516255436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516281283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516307208Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516328136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516342943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516361071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:14.516489022Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:14.516526986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:41357)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516551692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:14.516621082Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516650038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516672694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:36591)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516696411Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.516823312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.517350124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.51737649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.517391058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.517404636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.517927108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518022434Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518396599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518420385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518435113Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518537328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518555675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:14.518569323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.517682915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43036)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.517867037Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"io task","client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.518843732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.518877598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.519429056Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.519976884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.52000474Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[0]R flush_numbers[0..12]: [110, 82, 104, 110, 110, 97, 110, 110, 110, 92, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520021428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[0]R generation[0..12]: [56, 43, 54, 56, 56, 50, 56, 56, 56, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520039905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520054903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[1]R flush_numbers[0..12]: [110, 82, 104, 110, 110, 97, 110, 110, 110, 92, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520079939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[1]R generation[0..12]: [56, 43, 54, 56, 56, 50, 56, 56, 56, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520093737Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520107415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[2]R flush_numbers[0..12]: [110, 82, 104, 110, 110, 97, 110, 110, 110, 92, 106, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520120803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[2]R generation[0..12]: [56, 43, 54, 56, 56, 50, 56, 56, 56, 48, 54, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520134051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:20:24.520147819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Max found gen is 57","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520162737Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Generation requested: 58 >= found:57","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520176035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Next flush: 111","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520188753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520201541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"":"downstairs","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520215519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520227587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"0","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520241175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"1","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520256213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"client":"2","":"downstairs","session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520270071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520282639Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"f29be442-cdc0-479a-b8a6-f7f1b2f05d7a is now active with session: 96faf315-62e1-49a5-9608-6367454cff9d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520295267Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520306955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415,"session_id":"96faf315-62e1-49a5-9608-6367454cff9d"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:24.520319094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 90 len 16384 data: 6 4 5 6 02/30 Write block 129 len 32768 data: 5 5 5 5 5 6 6 5 03/30 Read block 291 len 4096 04/30 Write block 3 len 40960 data: 4 3 3 3 4 5 7 7 6 6 05/30 Flush 06/30 Write block 390 len 40960 data: 8 7 6 5 7 7 8 6 6 5 07/30 Read block 576 len 4096 08/30 Read block 183 len 36864 09/30 Read block 554 len 16384 10/30 Read block 476 len 36864 11/30 Read block 272 len 36864 12/30 Read block 411 len 12288 13/30 Read block 323 len 40960 14/30 Write block 304 len 36864 data: 3 3 3 2 2 3 4 5 6 15/30 Read block 280 len 12288 16/30 Read block 588 len 24576 17/30 Flush 18/30 Flush 19/30 Write block 434 len 16384 data: 5 4 5 5 20/30 Write block 373 len 24576 data: 9 9 9 6 6 8 21/30 Read block 489 len 32768 22/30 Read block 242 len 8192 23/30 Read block 396 len 24576 24/30 Read block 104 len 32768 25/30 Read block 177 len 16384 26/30 Read block 68 len 12288 27/30 Read block 580 len 20480 28/30 Write block 214 len 8192 data: 6 6 29/30 Read block 308 len 32768 30/30 Write block 342 len 20480 data: 7 7 6 7 7 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 4 Done Done Done false 20 Acked 1019 Write 6 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 2 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:24.544416182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1415} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 111 111 111 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 F F F 4 080-099 58 58 58 111 111 111 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 54 54 54 106 106 106 T T T 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 56 56 56 110 110 110 F F F 14 280-299 56 56 56 110 110 110 F F F 15 300-319 58 58 58 112 112 112 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 52 52 52 102 102 102 T T T 18 360-379 56 56 56 110 110 110 T T T 19 380-399 58 58 58 112 112 112 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 48 48 48 94 94 94 T T T 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 56 56 56 110 110 110 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 58, Max flush: 112 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 111 111 111 F F F 1 020-039 43 43 43 82 82 82 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 F F F 4 080-099 58 58 58 111 111 111 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 56 56 56 110 110 110 F F F 14 280-299 56 56 56 110 110 110 F F F 15 300-319 58 58 58 112 112 112 F F F 16 320-339 53 53 53 103 103 103 F F F 17 340-359 59 59 59 113 113 113 F F F 18 360-379 59 59 59 113 113 113 F F F 19 380-399 58 58 58 112 112 112 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 56 56 56 110 110 110 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 50 50 50 97 97 97 F F F Max gen: 59, Max flush: 113 Begin loop 30 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.752402518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.752868129Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.752895525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423} {"msg":"Crucible 79520998-010f-43c8-b5a3-3f6a53540c65 has session id: 43dd0509-b16a-4aee-9c1c-2f34bd4a94bf","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753056501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Upstairs opts: Upstairs UUID: 79520998-010f-43c8-b5a3-3f6a53540c65, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753190381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Crucible stats registered with UUID: 79520998-010f-43c8-b5a3-3f6a53540c65","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753214087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753544179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423} {"msg":"79520998-010f-43c8-b5a3-3f6a53540c65 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.75360434Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753643694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753672699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753698786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753724072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753749298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753773095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.7538028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753827267Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.753851053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:38.753963707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:38.754002061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:36890)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754029187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:38.754115534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754149939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754177175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:41079)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754314834Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.75434827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.75481398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754839877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754855454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.754869152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.755504258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.755531424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.75602797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.756051827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.756066205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.756130045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.756148623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:38.75616195Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.754891729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38353)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.754957399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"io task","client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.756118847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.756147103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.756595417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757142715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757169121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[0]R flush_numbers[0..12]: [111, 82, 104, 110, 111, 97, 111, 110, 110, 92, 113, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757190498Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[0]R generation[0..12]: [58, 43, 54, 56, 58, 50, 58, 56, 56, 48, 59, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757205286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757219824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[1]R flush_numbers[0..12]: [111, 82, 104, 110, 111, 97, 111, 110, 110, 92, 113, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757233812Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[1]R generation[0..12]: [58, 43, 54, 56, 58, 50, 58, 56, 56, 48, 59, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.75724746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757261778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[2]R flush_numbers[0..12]: [111, 82, 104, 110, 111, 97, 111, 110, 110, 92, 113, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757275446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[2]R generation[0..12]: [58, 43, 54, 56, 58, 50, 58, 56, 56, 48, 59, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757288974Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-08-05T19:20:48.757302692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757319879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757332827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757345555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757358244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"":"downstairs","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757371422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.75738335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"0","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757399667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"1","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757412825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"client":"2","":"downstairs","session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757426003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757437942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"79520998-010f-43c8-b5a3-3f6a53540c65 is now active with session: 43dd0509-b16a-4aee-9c1c-2f34bd4a94bf","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.75745052Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757462478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423,"session_id":"43dd0509-b16a-4aee-9c1c-2f34bd4a94bf"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:20:48.757474596Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 30 len 8192 data: 4 5 02/30 Read block 488 len 28672 03/30 Flush 04/30 Write block 580 len 28672 data: 7 6 4 3 2 3 4 05/30 Write block 377 len 32768 data: 7 9 9 10 8 6 5 5 06/30 Write block 331 len 36864 data: 6 5 5 5 5 7 7 7 7 07/30 Read block 450 len 24576 08/30 Read block 209 len 36864 09/30 Write block 590 len 24576 data: 5 5 5 5 5 6 10/30 Read block 173 len 24576 11/30 Read block 102 len 16384 12/30 Read block 562 len 8192 13/30 Read block 558 len 40960 14/30 Read block 588 len 36864 15/30 Read block 387 len 36864 16/30 Read block 482 len 28672 17/30 Write block 298 len 40960 data: 5 6 5 4 3 3 4 4 4 3 18/30 Read block 463 len 36864 19/30 Flush 20/30 Write block 264 len 8192 data: 5 6 21/30 Read block 365 len 36864 22/30 Read block 378 len 40960 23/30 Read block 110 len 32768 24/30 Read block 119 len 16384 25/30 Read block 558 len 12288 26/30 Read block 65 len 4096 27/30 Write block 283 len 40960 data: 6 6 5 7 9 7 7 9 8 7 28/30 Read block 445 len 8192 29/30 Read block 41 len 8192 30/30 Write block 267 len 36864 data: 5 5 6 5 6 6 6 7 7 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:20:48.80791028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1423} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 111 111 111 F F F 1 020-039 60 60 60 114 114 114 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 F F F 4 080-099 58 58 58 111 111 111 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 56 56 56 110 110 110 T T T 14 280-299 60 60 60 115 115 115 T T T 15 300-319 60 60 60 115 115 115 F F F 16 320-339 60 60 60 115 115 115 F F F 17 340-359 59 59 59 113 113 113 F F F 18 360-379 60 60 60 115 115 115 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 56 56 56 110 110 110 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 60 60 60 115 115 115 F F F Max gen: 60, Max flush: 115 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 58 58 58 111 111 111 F F F 1 020-039 60 60 60 114 114 114 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 F F F 4 080-099 58 58 58 111 111 111 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 58 58 58 111 111 111 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 51 51 51 101 101 101 F F F 12 240-259 46 46 46 87 87 87 F F F 13 260-279 61 61 61 116 116 116 F F F 14 280-299 61 61 61 116 116 116 F F F 15 300-319 60 60 60 115 115 115 F F F 16 320-339 60 60 60 115 115 115 F F F 17 340-359 59 59 59 113 113 113 F F F 18 360-379 60 60 60 115 115 115 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 54 54 54 104 104 104 F F F 26 520-539 56 56 56 110 110 110 F F F 27 540-559 54 54 54 105 105 105 F F F 28 560-579 54 54 54 104 104 104 F F F 29 580-599 60 60 60 115 115 115 F F F Max gen: 61, Max flush: 116 Begin loop 31 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.996881136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.997311972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.997343677Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431} {"msg":"Crucible e0e8a087-14ea-4d29-a32b-a01038d35576 has session id: a7c5b295-9495-40b1-98fc-ab2d89acc6f5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.997635894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Upstairs opts: Upstairs UUID: e0e8a087-14ea-4d29-a32b-a01038d35576, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.997684277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Crucible stats registered with UUID: e0e8a087-14ea-4d29-a32b-a01038d35576","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.997701114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.997972354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431} {"msg":"e0e8a087-14ea-4d29-a32b-a01038d35576 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998040724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.99806692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998083287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998097115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998110693Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998123701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998141249Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998154747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998168495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998194881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:02.998433975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:56067)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.99853647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:02.998565146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:55334)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998590872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:02.998621298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998643804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998666421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.998717533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.999339481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.999363458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.999377396Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:02.999395123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000018371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000139453Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000553201Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000582937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000605584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000658346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.000684602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:03.00069933Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.000001302Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47315)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.000195784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"io task","client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.001117957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.001145773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.00170205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002238371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002270496Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[0]R flush_numbers[0..12]: [111, 114, 104, 110, 111, 97, 111, 110, 110, 92, 113, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002287783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[0]R generation[0..12]: [58, 60, 54, 56, 58, 50, 58, 56, 56, 48, 59, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002302291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002316439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[1]R flush_numbers[0..12]: [111, 114, 104, 110, 111, 97, 111, 110, 110, 92, 113, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002330167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[1]R generation[0..12]: [58, 60, 54, 56, 58, 50, 58, 56, 56, 48, 59, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002343565Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002357663Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[2]R flush_numbers[0..12]: [111, 114, 104, 110, 111, 97, 111, 110, 110, 92, 113, 101]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002371271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[2]R generation[0..12]: [58, 60, 54, 56, 58, 50, 58, 56, 56, 48, 59, 51]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002384939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002398847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002411785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002425373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002438481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002451349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"":"downstairs","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:21:13.002464377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002482694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"0","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002496072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"1","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002509061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"client":"2","":"downstairs","session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002522229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002534357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"e0e8a087-14ea-4d29-a32b-a01038d35576 is now active with session: a7c5b295-9495-40b1-98fc-ab2d89acc6f5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002547005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002559543Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431,"session_id":"a7c5b295-9495-40b1-98fc-ab2d89acc6f5"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:13.002571791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 518 len 20480 data: 7 7 7 7 6 02/30 Write block 275 len 20480 data: 8 6 5 4 3 03/30 Read block 232 len 20480 04/30 Write block 572 len 24576 data: 6 5 4 4 4 4 05/30 Read block 431 len 20480 06/30 Read block 172 len 8192 07/30 Read block 293 len 28672 08/30 Read block 193 len 36864 09/30 Write block 16 len 36864 data: 5 4 3 4 4 4 5 5 5 10/30 Write block 373 len 12288 data: 10 10 10 11/30 Read block 56 len 16384 12/30 Flush 13/30 Write block 315 len 40960 data: 8 7 7 5 4 4 4 4 5 6 14/30 Flush 15/30 Write block 126 len 32768 data: 3 5 4 6 6 6 6 6 16/30 Read block 568 len 16384 17/30 Flush 18/30 Write block 556 len 36864 data: 6 4 4 4 5 7 6 8 7 19/30 Write block 345 len 40960 data: 8 8 5 6 6 6 5 6 5 5 20/30 Read block 558 len 24576 21/30 Read block 509 len 28672 22/30 Write block 568 len 12288 data: 6 4 6 23/30 Read block 61 len 40960 24/30 Read block 541 len 36864 25/30 Write block 62 len 24576 data: 5 4 5 5 4 4 26/30 Read block 72 len 20480 27/30 Read block 85 len 16384 28/30 Write block 183 len 12288 data: 5 4 4 29/30 Write block 3 len 16384 data: 5 4 4 4 30/30 Write block 237 len 32768 data: 4 5 5 5 3 4 3 4 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 9 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 6 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 9 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:13.030641631Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1431} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 62 62 62 117 117 117 T T T 1 020-039 62 62 62 117 117 117 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 56 56 56 110 110 110 T T T 4 080-099 58 58 58 111 111 111 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 62 62 62 119 119 119 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 48 48 48 92 92 92 T T T 10 200-219 59 59 59 113 113 113 F F F 11 220-239 51 51 51 101 101 101 T T T 12 240-259 46 46 46 87 87 87 T T T 13 260-279 62 62 62 117 117 117 F F F 14 280-299 61 61 61 116 116 116 F F F 15 300-319 62 62 62 118 118 118 F F F 16 320-339 62 62 62 118 118 118 F F F 17 340-359 59 59 59 113 113 113 T T T 18 360-379 62 62 62 117 117 117 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 62 62 62 117 117 117 F F F 26 520-539 62 62 62 117 117 117 F F F 27 540-559 54 54 54 105 105 105 T T T 28 560-579 62 62 62 117 117 117 T T T 29 580-599 60 60 60 115 115 115 F F F Max gen: 62, Max flush: 119 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 120 120 120 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 63 63 63 120 120 120 F F F 4 080-099 58 58 58 111 111 111 F F F 5 100-119 50 50 50 97 97 97 F F F 6 120-139 62 62 62 119 119 119 F F F 7 140-159 56 56 56 110 110 110 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 63 63 63 120 120 120 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 63 63 63 120 120 120 F F F 12 240-259 63 63 63 120 120 120 F F F 13 260-279 62 62 62 117 117 117 F F F 14 280-299 61 61 61 116 116 116 F F F 15 300-319 62 62 62 118 118 118 F F F 16 320-339 62 62 62 118 118 118 F F F 17 340-359 63 63 63 120 120 120 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 54 54 54 105 105 105 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 62 62 62 117 117 117 F F F 26 520-539 62 62 62 117 117 117 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 60 60 60 115 115 115 F F F Max gen: 63, Max flush: 120 Begin loop 32 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.284563159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285215073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285246238Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439} {"msg":"Crucible 492aa6c6-b9f6-49d0-af27-f6d434df6119 has session id: d6660b06-165b-4691-ae17-360d2dae40e6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285269954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Upstairs opts: Upstairs UUID: 492aa6c6-b9f6-49d0-af27-f6d434df6119, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285326756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Crucible stats registered with UUID: 492aa6c6-b9f6-49d0-af27-f6d434df6119","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285355182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285731206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439} {"msg":"492aa6c6-b9f6-49d0-af27-f6d434df6119 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285756542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.28577095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285786078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285799906Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285817093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285830531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285843799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285913209Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285944434Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.285969901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44279)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286132976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:27.286172851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:27.286202656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:27.286304371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54181)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286336546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286363502Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286386769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286407786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286928068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286958784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.286974472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.28698841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.287565294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.287706003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.288113583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.288137619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.288151657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.288238544Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.288256312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:27.28826985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.288292916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:55545)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.288353587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"io task","client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.289327422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.289355488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.289933193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290515656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290543332Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[0]R flush_numbers[0..12]: [120, 117, 104, 120, 111, 97, 119, 110, 110, 120, 113, 120]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.29056002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[0]R generation[0..12]: [63, 62, 54, 63, 58, 50, 62, 56, 56, 63, 59, 63]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290574838Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290588955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[1]R flush_numbers[0..12]: [120, 117, 104, 120, 111, 97, 119, 110, 110, 120, 113, 120]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290602983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[1]R generation[0..12]: [63, 62, 54, 63, 58, 50, 62, 56, 56, 63, 59, 63]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290616781Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290630609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[2]R flush_numbers[0..12]: [120, 117, 104, 120, 111, 97, 119, 110, 110, 120, 113, 120]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290644107Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[2]R generation[0..12]: [63, 62, 54, 63, 58, 50, 62, 56, 56, 63, 59, 63]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290657535Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290671373Z","hostnameWait for a query_work_queue command to finish before sending IO ":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290687891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290700939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290717076Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290730204Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"":"downstairs","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290743422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290755461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"0","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290768599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"1","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290781477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"client":"2","":"downstairs","session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290794635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290806753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"492aa6c6-b9f6-49d0-af27-f6d434df6119 is now active with session: d6660b06-165b-4691-ae17-360d2dae40e6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290819491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.29083132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439,"session_id":"d6660b06-165b-4691-ae17-360d2dae40e6"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:37.290843458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 239 len 36864 data: 6 6 4 5 4 5 5 5 4 02/30 Write block 284 len 32768 data: 7 6 8 10 8 8 10 9 03/30 Read block 371 len 36864 04/30 Write block 142 len 32768 data: 6 7 8 8 6 4 4 4 05/30 Write block 442 len 8192 data: 8 7 06/30 Read block 158 len 40960 07/30 Read block 51 len 16384 08/30 Read block 331 len 40960 09/30 Write block 254 len 24576 data: 8 6 6 7 7 6 10/30 Read block 204 len 32768 11/30 Read block 147 len 16384 12/30 Read block 238 len 12288 13/30 Read block 213 len 20480 14/30 Read block 387 len 20480 15/30 Read block 190 len 8192 16/30 Read block 509 len 16384 17/30 Read block 108 len 40960 18/30 Read block 589 len 12288 19/30 Read block 275 len 24576 20/30 Read block 452 len 24576 21/30 Write block 5 len 32768 data: 5 5 5 6 8 8 7 7 22/30 Write block 333 len 20480 data: 6 6 6 8 8 23/30 Write block 106 len 8192 data: 7 6 24/30 Write block 232 len 16384 data: 4 4 4 4 25/30 Write block 69 len 4096 data: 5 26/30 Read block 440 len 16384 27/30 Write block 273 len 28672 data: 7 8 9 7 6 5 4 28/30 Read block 365 len 16384 29/30 Write block 249 len 4096 data: 6 30/30 Write block 95 len 32768 data: 6 5 4 3 2 3 3 3 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 1 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:37.313209632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1439} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 63 63 63 120 120 120 T T T 4 080-099 58 58 58 111 111 111 T T T 5 100-119 50 50 50 97 97 97 T T T 6 120-139 62 62 62 119 119 119 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 63 63 63 120 120 120 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 64 64 64 121 121 121 T T T 12 240-259 64 64 64 121 121 121 T T T 13 260-279 62 62 62 117 117 117 T T T 14 280-299 64 64 64 121 121 121 F F F 15 300-319 62 62 62 118 118 118 F F F 16 320-339 62 62 62 118 118 118 T T T 17 340-359 63 63 63 120 120 120 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 62 62 62 117 117 117 F F F 26 520-539 62 62 62 117 117 117 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 60 60 60 115 115 115 F F F Max gen: 64, Max flush: 121 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 54 54 54 104 104 104 F F F 3 060-079 65 65 65 122 122 122 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 65 65 65 122 122 122 F F F 6 120-139 62 62 62 119 119 119 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 63 63 63 120 120 120 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 65 65 65 122 122 122 F F F 12 240-259 65 65 65 122 122 122 F F F 13 260-279 65 65 65 122 122 122 F F F 14 280-299 64 64 64 121 121 121 F F F 15 300-319 62 62 62 118 118 118 F F F 16 320-339 65 65 65 122 122 122 F F F 17 340-359 63 63 63 120 120 120 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 60 60 60 115 115 115 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 59 59 59 113 113 113 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 56 56 56 110 110 110 F F F 24 480-499 56 56 56 110 110 110 F F F 25 500-519 62 62 62 117 117 117 F F F 26 520-539 62 62 62 117 117 117 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 60 60 60 115 115 115 F F F Max gen: 65, Max flush: 122 Begin loop 33 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.55296633Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.553409515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.553436971Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447} {"msg":"Crucible 91651a44-3c91-4191-a5c8-ffab20757274 has session id: 58195764-e7e5-4686-ac5f-66e555b9f0d4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.553678855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Upstairs opts: Upstairs UUID: 91651a44-3c91-4191-a5c8-ffab20757274, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.553727928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Crucible stats registered with UUID: 91651a44-3c91-4191-a5c8-ffab20757274","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.553754884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554042121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447} {"msg":"91651a44-3c91-4191-a5c8-ffab20757274 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554066557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554080665Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554095443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.55411294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554126938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554145366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554169782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.55418577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554199478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554245091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:34185)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554556345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:42606)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554594739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:51.554625315Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:51.55465354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:21:51.554681736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554704543Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554729419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.554818296Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.555361115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.555386822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.555401149Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.555419847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556035306Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556064111Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556552839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556590763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556619539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556645415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556673241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:21:51.556695608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.556288388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49480)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.556402791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"io task","client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.557431339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.557458915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558009203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558561321Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558588917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 104, 122, 122, 122, 119, 121, 110, 120, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558605565Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[0]R generation[0..12]: [64, 62, 54, 65, 65, 65, 62, 64, 56, 63, 59, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558620752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 104, 122, 122, 122, 119, 121, 110, 120, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558649088Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[1]R generation[0..12]: [64, 62, 54, 65, 65, 65, 62, 64, 56, 63, 59, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558662936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558677024Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 104, 122, 122, 122, 119, 121, 110, 120, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558690782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[2]R generation[0..12]: [64, 62, 54, 65, 65, 65, 62, 64, 56, 63, 59, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.55870424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558718198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558731106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558751903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558764641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558777679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"":"downstairs","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558790747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558802765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"0","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558815774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"1","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558828621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"client":"2","":"downstairs","session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.55884229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558854788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"91651a44-3c91-4191-a5c8-ffab20757274 is now active with session: 58195764-e7e5-4686-ac5f-66e555b9f0d4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558867566Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558879334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447,"session_id":"58195764-e7e5-4686-ac5f-66e555b9f0d4"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:01.558891342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 8192 data: 4 5 02/30 Write block 294 len 28672 data: 6 7 7 7 6 7 6 03/30 Read block 34 len 24576 04/30 Read block 376 len 32768 05/30 Read block 148 len 32768 06/30 Write block 466 len 12288 data: 5 8 4 07/30 Write block 129 len 8192 data: 7 7 08/30 Flush 09/30 Flush 10/30 Write block 495 len 36864 data: 4 5 6 4 3 3 2 2 3 11/30 Flush 12/30 Write block 162 len 4096 data: 11 13/30 Flush 14/30 Write block 44 len 12288 data: 7 6 5 15/30 Write block 422 len 20480 data: 3 5 4 5 4 16/30 Write block 393 len 8192 data: 6 8 17/30 Read block 349 len 32768 18/30 Write block 342 len 20480 data: 8 8 7 9 9 19/30 Read block 448 len 40960 20/30 Read block 141 len 28672 21/30 Read block 107 len 20480 22/30 Read block 565 len 12288 23/30 Write block 581 len 40960 data: 7 5 4 3 4 5 4 4 4 6 24/30 Write block 248 len 28672 data: 4 7 7 8 9 10 9 25/30 Write block 316 len 4096 data: 8 26/30 Read block 414 len 32768 27/30 Read block 445 len 36864 28/30 Read block 335 len 20480 29/30 Read block 137 len 20480 30/30 Read block 395 len 40960 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Write 10 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Read 10 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 9 9 10 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:01.587103661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1447} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 66 66 66 127 127 127 F F F 3 060-079 65 65 65 122 122 122 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 65 65 65 122 122 122 F F F 6 120-139 66 66 66 123 123 123 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 66 66 66 126 126 126 F F F 9 180-199 63 63 63 120 120 120 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 65 65 65 122 122 122 F F F 12 240-259 65 65 65 122 122 122 T T T 13 260-279 65 65 65 122 122 122 F F F 14 280-299 66 66 66 123 123 123 F F F 15 300-319 66 66 66 123 123 123 T T T 16 320-339 65 65 65 122 122 122 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 66 66 66 127 127 127 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 66 66 66 125 125 125 F F F 25 500-519 66 66 66 125 125 125 F F F 26 520-539 62 62 62 117 117 117 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 60 60 60 115 115 115 T T T Max gen: 66, Max flush: 127 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 66 66 66 127 127 127 F F F 3 060-079 65 65 65 122 122 122 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 65 65 65 122 122 122 F F F 6 120-139 66 66 66 123 123 123 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 66 66 66 126 126 126 F F F 9 180-199 63 63 63 120 120 120 F F F 10 200-219 59 59 59 113 113 113 F F F 11 220-239 65 65 65 122 122 122 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 65 65 65 122 122 122 F F F 14 280-299 66 66 66 123 123 123 F F F 15 300-319 67 67 67 128 128 128 F F F 16 320-339 65 65 65 122 122 122 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 52 52 52 102 102 102 F F F 21 420-439 66 66 66 127 127 127 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 66 66 66 125 125 125 F F F 25 500-519 66 66 66 125 125 125 F F F 26 520-539 62 62 62 117 117 117 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 67 67 67 128 128 128 F F F Max gen: 67, Max flush: 128 Begin loop 34 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.781097752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.781539176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.781574101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455} {"msg":"Crucible 41694b07-6234-495f-b79c-222a9d8d9630 has session id: 41ee6595-77e6-4ab7-a0b5-42ee2891435c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.781721429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Upstairs opts: Upstairs UUID: 41694b07-6234-495f-b79c-222a9d8d9630, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.781765683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Crucible stats registered with UUID: 41694b07-6234-495f-b79c-222a9d8d9630","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.781790439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782166143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455} {"msg":"41694b07-6234-495f-b79c-222a9d8d9630 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782262918Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782281895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782297323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.78231443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782334118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782358714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.78238315Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782401928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782415926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782429404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55540)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782672697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:15.782714291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:15.782745706Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:15.782840683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782864799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782888345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.782907753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63919)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.78292664Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.783360755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.78339469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.783418237Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.783440754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784047004Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784086668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784552129Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784578425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784593203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784605961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784619599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:15.784632787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.784569576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36515)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.784739031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"io task","client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.785536573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.785571098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786102929Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786643049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786670805Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 127, 122, 122, 122, 123, 121, 126, 120, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786687482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[0]R generation[0..12]: [64, 62, 66, 65, 65, 65, 66, 64, 66, 63, 59, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.78670205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786716388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 127, 122, 122, 122, 123, 121, 126, 120, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786730266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[1]R generation[0..12]: [64, 62, 66, 65, 65, 65, 66, 64, 66, 63, 59, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786743684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786757512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 127, 122, 122, 122, 123, 121, 126, 120, 113, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.78677123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[2]R generation[0..12]: [64, 62, 66, 65, 65, 65, 66, 64, 66, 63, 59, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786784638Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:22:25.786798706Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786815174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786828761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Next flush: 129","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.7868416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786854338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"":"downstairs","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786867416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786879344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"0","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786892652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"1","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.78690637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"client":"2","":"downstairs","session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786919868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786932106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"41694b07-6234-495f-b79c-222a9d8d9630 is now active with session: 41ee6595-77e6-4ab7-a0b5-42ee2891435c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786944954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786956882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455,"session_id":"41ee6595-77e6-4ab7-a0b5-42ee2891435c"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:25.786968901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 185 len 36864 data: 5 5 5 5 5 5 5 4 3 02/30 Read block 597 len 8192 03/30 Read block 560 len 8192 04/30 Write block 61 len 12288 data: 7 6 5 05/30 Read block 315 len 8192 06/30 Read block 589 len 16384 07/30 Flush 08/30 Write block 409 len 40960 data: 6 6 7 6 5 4 4 5 5 4 09/30 Write block 424 len 20480 data: 5 6 5 6 5 10/30 Read block 390 len 12288 11/30 Read block 19 len 4096 12/30 Flush 13/30 Flush 14/30 Write block 405 len 32768 data: 9 8 8 6 7 7 8 7 15/30 Write block 51 len 12288 data: 4 5 5 16/30 Write block 431 len 16384 data: 6 6 6 6 17/30 Read block 216 len 28672 18/30 Read block 284 len 4096 19/30 Write block 532 len 16384 data: 5 6 6 5 20/30 Write block 204 len 24576 data: 4 4 6 8 6 6 21/30 Write block 512 len 12288 data: 6 3 6 22/30 Read block 110 len 16384 23/30 Read block 250 len 24576 24/30 Read block 348 len 4096 25/30 Flush 26/30 Write block 291 len 8192 data: 10 8 27/30 Read block 339 len 12288 28/30 Read block 128 len 16384 29/30 Read block 528 len 32768 30/30 Read block 516 len 4096 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:25.815118828Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1455} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 68 68 68 132 132 132 F F F 3 060-079 68 68 68 129 129 129 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 65 65 65 122 122 122 F F F 6 120-139 66 66 66 123 123 123 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 66 66 66 126 126 126 F F F 9 180-199 68 68 68 129 129 129 F F F 10 200-219 68 68 68 133 133 133 F F F 11 220-239 65 65 65 122 122 122 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 65 65 65 122 122 122 F F F 14 280-299 66 66 66 123 123 123 T T T 15 300-319 67 67 67 128 128 128 F F F 16 320-339 65 65 65 122 122 122 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 68 68 68 132 132 132 F F F 21 420-439 68 68 68 133 133 133 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 66 66 66 125 125 125 F F F 25 500-519 68 68 68 133 133 133 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 67 67 67 128 128 128 F F F Max gen: 68, Max flush: 133 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 68 68 68 132 132 132 F F F 3 060-079 68 68 68 129 129 129 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 65 65 65 122 122 122 F F F 6 120-139 66 66 66 123 123 123 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 66 66 66 126 126 126 F F F 9 180-199 68 68 68 129 129 129 F F F 10 200-219 68 68 68 133 133 133 F F F 11 220-239 65 65 65 122 122 122 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 65 65 65 122 122 122 F F F 14 280-299 69 69 69 134 134 134 F F F 15 300-319 67 67 67 128 128 128 F F F 16 320-339 65 65 65 122 122 122 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 68 68 68 132 132 132 F F F 21 420-439 68 68 68 133 133 133 F F F 22 440-459 64 64 64 121 121 121 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 66 66 66 125 125 125 F F F 25 500-519 68 68 68 133 133 133 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 67 67 67 128 128 128 F F F Max gen: 69, Max flush: 134 Begin loop 35 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.984644865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985435668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985466784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463} {"msg":"Crucible 6ae25c5b-a108-400e-8bf4-0845e168a1e1 has session id: 6138650e-47d2-4c4e-bd7b-0dd1134db624","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.98549083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Upstairs opts: Upstairs UUID: 6ae25c5b-a108-400e-8bf4-0845e168a1e1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985526015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Crucible stats registered with UUID: 6ae25c5b-a108-400e-8bf4-0845e168a1e1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985551811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985723566Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463} {"msg":"6ae25c5b-a108-400e-8bf4-0845e168a1e1 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985877912Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.98589654Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985917337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985946572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985972809Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.985995505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986020881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986043748Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986066684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986088561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_connection connected from Ok(127.0.0.1:47882)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986277073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:39.986456667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_connection connected from Ok(127.0.0.1:39244)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986486072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:39.98650326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:39.986579728Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986598796Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986612534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.986728246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.987284694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.98731274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.987327228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.987348865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.987969873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.987997248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.988487696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.988511592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.98852545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.988610067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.988627685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:39.988640883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.98792308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_connection connected from Ok(127.0.0.1:35654)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.988096484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"io task","client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.988983532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.989028736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.989692178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990302907Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990332472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 132, 129, 122, 122, 123, 121, 126, 129, 133, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990346741Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[0]R generation[0..12]: [64, 62, 68, 68, 65, 65, 66, 64, 66, 68, 68, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990360788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990375046Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 132, 129, 122, 122, 123, 121, 126, 129, 133, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990388854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[1]R generation[0..12]: [64, 62, 68, 68, 65, 65, 66, 64, 66, 68, 68, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990402222Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.99041605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 132, 129, 122, 122, 123, 121, 126, 129, 133, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990430058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[2]R generation[0..12]: [64, 62, 68, 68, 65, 65, 66, 64, 66, 68, 68, 65]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990443556Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990457324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990470012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.9904865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Next flush: 135","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990499598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":Wait for a query_work_queue command to finish before sending IO "All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990516005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"":"downstairs","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990529323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990541242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"0","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.99055404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"1","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990567018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"client":"2","":"downstairs","session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990580006Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990591894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"6ae25c5b-a108-400e-8bf4-0845e168a1e1 is now active with session: 6138650e-47d2-4c4e-bd7b-0dd1134db624","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990604542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.99061635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463,"session_id":"6138650e-47d2-4c4e-bd7b-0dd1134db624"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:22:49.990628909Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 204 len 24576 data: 5 5 7 9 7 7 02/30 Write block 297 len 36864 data: 8 7 8 7 5 4 4 5 5 03/30 Write block 166 len 4096 data: 10 04/30 Flush 05/30 Write block 169 len 28672 data: 9 7 8 7 8 7 7 06/30 Read block 400 len 12288 07/30 Read block 451 len 32768 08/30 Read block 315 len 40960 09/30 Read block 149 len 40960 10/30 Read block 315 len 28672 11/30 Read block 403 len 24576 12/30 Read block 71 len 24576 13/30 Flush 14/30 Write block 112 len 8192 data: 11 11 15/30 Write block 483 len 28672 data: 6 8 8 7 8 11 11 16/30 Write block 448 len 36864 data: 7 7 7 6 6 7 6 5 5 17/30 Write block 416 len 8192 data: 6 6 18/30 Read block 354 len 12288 19/30 Read block 529 len 4096 20/30 Read block 432 len 40960 21/30 Write block 414 len 12288 data: 5 5 7 22/30 Read block 383 len 32768 23/30 Write block 215 len 32768 data: 7 6 6 5 6 7 5 3 24/30 Flush 25/30 Flush 26/30 Write block 510 len 28672 data: 7 7 7 4 7 6 6 27/30 Write block 586 len 12288 data: 6 5 5 28/30 Write block 330 len 20480 data: 6 7 6 7 7 29/30 Write block 276 len 16384 data: 8 7 6 5 30/30 Read block 216 len 12288 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Write 4 Done Done Done false 31 Acked 1030 Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:22:50.0190883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1463} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 68 68 68 132 132 132 F F F 3 060-079 68 68 68 129 129 129 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 70 70 70 137 137 137 F F F 6 120-139 66 66 66 123 123 123 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 70 70 70 136 136 136 F F F 9 180-199 68 68 68 129 129 129 F F F 10 200-219 70 70 70 138 138 138 F F F 11 220-239 70 70 70 138 138 138 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 65 65 65 122 122 122 T T T 14 280-299 70 70 70 135 135 135 F F F 15 300-319 70 70 70 135 135 135 F F F 16 320-339 65 65 65 122 122 122 T T T 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 68 68 68 133 133 133 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 70 70 70 137 137 137 F F F 25 500-519 68 68 68 133 133 133 T T T 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 67 67 67 128 128 128 T T T Max gen: 70, Max flush: 138 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 68 68 68 132 132 132 F F F 3 060-079 68 68 68 129 129 129 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 70 70 70 137 137 137 F F F 6 120-139 66 66 66 123 123 123 F F F 7 140-159 64 64 64 121 121 121 F F F 8 160-179 70 70 70 136 136 136 F F F 9 180-199 68 68 68 129 129 129 F F F 10 200-219 70 70 70 138 138 138 F F F 11 220-239 70 70 70 138 138 138 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 71 71 71 139 139 139 F F F 14 280-299 70 70 70 135 135 135 F F F 15 300-319 70 70 70 135 135 135 F F F 16 320-339 71 71 71 139 139 139 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 68 68 68 133 133 133 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 66 66 66 123 123 123 F F F 24 480-499 70 70 70 137 137 137 F F F 25 500-519 71 71 71 139 139 139 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 63 63 63 120 120 120 F F F 29 580-599 71 71 71 139 139 139 F F F Max gen: 71, Max flush: 139 Begin loop 36 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.228419996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.228675998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.228692156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471} {"msg":"Crucible 42a0c1dc-5e0d-455f-9df6-b9592344d118 has session id: 19e7355a-181d-45ba-9b73-44aecad9c7ec","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229305175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Upstairs opts: Upstairs UUID: 42a0c1dc-5e0d-455f-9df6-b9592344d118, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.2293407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Crucible stats registered with UUID: 42a0c1dc-5e0d-455f-9df6-b9592344d118","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229356548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229713285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471} {"msg":"42a0c1dc-5e0d-455f-9df6-b9592344d118 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.2297468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229760808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229775306Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229792273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229806221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229819819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229833287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229852464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.229866242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.22987959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:04.230092259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:04.230131693Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:04.230159549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230180686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:46326)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230202542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230227859Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230329993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:56930)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230376217Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230799574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.23082575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230929135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.230956431Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231314837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231420282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231831491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231858637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231873844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231914948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231931876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:04.231945354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.231251387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_connection connected from Ok(127.0.0.1:48071)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.231440859Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"io task","client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.232339196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.232371471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.232943656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233476837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233512302Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 132, 129, 122, 137, 123, 121, 136, 129, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233537988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[0]R generation[0..12]: [64, 62, 68, 68, 65, 70, 66, 64, 70, 68, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233564184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233588921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 132, 129, 122, 137, 123, 121, 136, 129, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233612287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[1]R generation[0..12]: [64, 62, 68, 68, 65, 70, 66, 64, 70, 68, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233634894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.23365929Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 132, 129, 122, 137, 123, 121, 136, 129, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233685616Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"[2]R generation[0..12]: [64, 62, 68, 68, 65, 70, 66, 64, 70, 68, 70, 70]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233710552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233738978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233764675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.23379477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Next flush: 140","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233822496Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233850012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"":"downstairs","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233877108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233901744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"0","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.23392792Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"1","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233953676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"client":"2","":"downstairs","session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.233980752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.234004619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"42a0c1dc-5e0d-455f-9df6-b9592344d118 is now active with session: 19e7355a-181d-45ba-9b73-44aecad9c7ec","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.234029405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.234054092Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471,"session_id":"19e7355a-181d-45ba-9b73-44aecad9c7ec"Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:14.234085327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 191 len 12288 data: 6 5 4 03/30 Read block 585 len 12288 04/30 Flush 05/30 Write block 159 len 16384 data: 7 10 9 12 06/30 Flush 07/30 Write block 219 len 20480 data: 7 8 6 4 3 08/30 Read block 55 len 28672 09/30 Write block 465 len 16384 data: 4 6 9 5 10/30 Read block 166 len 32768 11/30 Read block 357 len 28672 12/30 Read block 13 len 36864 13/30 Read block 456 len 8192 14/30 Write block 568 len 20480 data: 7 5 7 8 7 15/30 Read block 185 len 28672 16/30 Write block 303 len 12288 data: 5 6 6 17/30 Read block 247 len 36864 18/30 Read block 539 len 28672 19/30 Read block 344 len 4096 20/30 Read block 133 len 24576 21/30 Read block 476 len 4096 22/30 Read block 97 len 8192 23/30 Write block 73 len 20480 data: 4 4 3 3 4 24/30 Write block 591 len 24576 data: 6 6 6 6 7 5 25/30 Write block 150 len 32768 data: 4 4 4 5 6 6 9 9 26/30 Write block 211 len 32768 data: 9 8 7 7 8 7 7 6 27/30 Write block 136 len 40960 data: 6 6 5 5 5 6 7 8 9 9 28/30 Write block 436 len 4096 data: 6 29/30 Read block 192 len 8192 30/30 Read block 44 len 20480 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Read 1 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false 31 Acked 1030 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:14.261679587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1471} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 68 68 68 132 132 132 F F F 3 060-079 68 68 68 129 129 129 T T T 4 080-099 65 65 65 122 122 122 F F F 5 100-119 70 70 70 137 137 137 F F F 6 120-139 66 66 66 123 123 123 T T T 7 140-159 72 72 72 142 142 142 T T T 8 160-179 72 72 72 142 142 142 F F F 9 180-199 72 72 72 141 141 141 F F F 10 200-219 72 72 72 143 143 143 T T T 11 220-239 72 72 72 143 143 143 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 71 71 71 139 139 139 F F F 14 280-299 70 70 70 135 135 135 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 71 71 71 139 139 139 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 68 68 68 133 133 133 T T T 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 70 70 70 137 137 137 F F F 25 500-519 71 71 71 139 139 139 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 72 72 72 143 143 143 F F F 29 580-599 71 71 71 139 139 139 T T T Max gen: 72, Max flush: 143 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 68 68 68 132 132 132 F F F 3 060-079 73 73 73 144 144 144 F F F 4 080-099 65 65 65 122 122 122 F F F 5 100-119 70 70 70 137 137 137 F F F 6 120-139 73 73 73 144 144 144 F F F 7 140-159 73 73 73 144 144 144 F F F 8 160-179 72 72 72 142 142 142 F F F 9 180-199 72 72 72 141 141 141 F F F 10 200-219 73 73 73 144 144 144 F F F 11 220-239 72 72 72 143 143 143 F F F 12 240-259 67 67 67 128 128 128 F F F 13 260-279 71 71 71 139 139 139 F F F 14 280-299 70 70 70 135 135 135 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 71 71 71 139 139 139 F F F 17 340-359 66 66 66 127 127 127 F F F 18 360-379 62 62 62 117 117 117 F F F 19 380-399 66 66 66 127 127 127 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 73 73 73 144 144 144 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 70 70 70 137 137 137 F F F 25 500-519 71 71 71 139 139 139 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 72 72 72 143 143 143 F F F 29 580-599 73 73 73 144 144 144 F F F Max gen: 73, Max flush: 144 Begin loop 37 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.488361684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.488829254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.48885658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479} {"msg":"Crucible 9f264aca-4103-48f5-8b73-37fbec1fe55b has session id: adf19ea2-1c9f-4b43-b366-de52e4a9761a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489017696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Upstairs opts: Upstairs UUID: 9f264aca-4103-48f5-8b73-37fbec1fe55b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489090346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Crucible stats registered with UUID: 9f264aca-4103-48f5-8b73-37fbec1fe55b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.48912701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489412478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479} {"msg":"9f264aca-4103-48f5-8b73-37fbec1fe55b active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489435714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489450582Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.48946943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489489427Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489514293Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489541879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489567625Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489591351Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489616078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489638584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:28.489813518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_connection connected from Ok(127.0.0.1:51619)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.48993641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:28.489965626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_connection connected from Ok(127.0.0.1:56410)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.489989802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:28.490017258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490037955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490065531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490088798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490523353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490555318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490577885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.490599312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.491062603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.491177196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.491598744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.49162667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.491642107Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.491667824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.491682162Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:28.49169516Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.49135276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_connection connected from Ok(127.0.0.1:48016)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.491506548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"io task","client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.492506199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.492532395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493094772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493660358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493682285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[0]R flush_numbers[0..12]: [121, 117, 132, 144, 122, 137, 144, 144, 142, 141, 144, 143]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493696863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[0]R generation[0..12]: [64, 62, 68, 73, 65, 70, 73, 73, 72, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493711241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493725508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[1]R flush_numbers[0..12]: [121, 117, 132, 144, 122, 137, 144, 144, 142, 141, 144, 143]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493739206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[1]R generation[0..12]: [64, 62, 68, 73, 65, 70, 73, 73, 72, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493752595Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493766422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[2]R flush_numbers[0..12]: [121, 117, 132, 144, 122, 137, 144, 144, 142, 141, 144, 143]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493780291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[2]R generation[0..12]: [64, 62, 68, 73, 65, 70, 73, 73, 72, 72, 73, 72]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493793589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493807346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493823794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493837302Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Wait for a query_work_queue command to finish before sending IO Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.49385382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493867098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"":"downstairs","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493880146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493891954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"0","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493905012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"1","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.49391784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"client":"2","":"downstairs","session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493930828Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493942846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"9f264aca-4103-48f5-8b73-37fbec1fe55b is now active with session: adf19ea2-1c9f-4b43-b366-de52e4a9761a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493955545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.493967503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479,"session_id":"adf19ea2-1c9f-4b43-b366-de52e4a9761a"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:38.49398774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 8192 data: 6 5 02/30 Read block 22 len 24576 03/30 Flush 04/30 Write block 146 len 20480 data: 7 5 5 5 5 05/30 Flush 06/30 Write block 220 len 8192 data: 9 7 07/30 Flush 08/30 Write block 53 len 28672 data: 6 5 7 7 7 6 7 09/30 Write block 389 len 32768 data: 8 9 8 7 7 9 8 9 10/30 Write block 385 len 36864 data: 4 5 6 7 9 10 9 8 8 11/30 Write block 205 len 36864 data: 6 8 10 8 8 8 10 9 8 12/30 Write block 258 len 32768 data: 8 7 5 6 6 5 6 7 13/30 Read block 358 len 20480 14/30 Write block 367 len 20480 data: 10 8 7 6 6 15/30 Write block 562 len 32768 data: 7 9 8 8 6 6 8 6 16/30 Read block 85 len 12288 17/30 Write block 366 len 28672 data: 9 11 9 8 7 7 6 18/30 Write block 88 len 24576 data: 6 7 7 5 6 7 19/30 Read block 207 len 28672 20/30 Read block 318 len 16384 21/30 Read block 282 len 36864 22/30 Read block 270 len 4096 23/30 Read block 427 len 8192 24/30 Write block 340 len 32768 data: 6 6 9 9 8 10 10 6 25/30 Read block 86 len 28672 26/30 Read block 267 len 16384 27/30 Write block 497 len 20480 data: 7 5 4 4 3 28/30 Write block 10 len 32768 data: 9 8 8 5 4 6 6 5 29/30 Write block 166 len 20480 data: 11 10 8 10 8 30/30 Read block 345 len 36864 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 8 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Write 7 Done Done Done false 19 Acked 1018 Write 6 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 4 Done Done Done false 22 Acked 1021 Read 9 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Write 5 Done Done Done false 31 Acked 1030 Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:38.522688895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1479} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 64 64 64 121 121 121 T T T 1 020-039 62 62 62 117 117 117 F F F 2 040-059 74 74 74 148 148 148 F F F 3 060-079 73 73 73 144 144 144 F F F 4 080-099 65 65 65 122 122 122 T T T 5 100-119 70 70 70 137 137 137 F F F 6 120-139 73 73 73 144 144 144 F F F 7 140-159 74 74 74 146 146 146 F F F 8 160-179 72 72 72 142 142 142 T T T 9 180-199 72 72 72 141 141 141 F F F 10 200-219 74 74 74 148 148 148 F F F 11 220-239 74 74 74 147 147 147 F F F 12 240-259 74 74 74 148 148 148 F F F 13 260-279 74 74 74 148 148 148 F F F 14 280-299 70 70 70 135 135 135 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 71 71 71 139 139 139 F F F 17 340-359 66 66 66 127 127 127 T T T 18 360-379 74 74 74 148 148 148 T T T 19 380-399 74 74 74 148 148 148 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 73 73 73 144 144 144 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 70 70 70 137 137 137 T T T 25 500-519 71 71 71 139 139 139 T T T 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 72 72 72 143 143 143 T T T 29 580-599 73 73 73 144 144 144 F F F Max gen: 74, Max flush: 148 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 149 149 149 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 74 74 74 148 148 148 F F F 3 060-079 73 73 73 144 144 144 F F F 4 080-099 75 75 75 149 149 149 F F F 5 100-119 70 70 70 137 137 137 F F F 6 120-139 73 73 73 144 144 144 F F F 7 140-159 74 74 74 146 146 146 F F F 8 160-179 75 75 75 149 149 149 F F F 9 180-199 72 72 72 141 141 141 F F F 10 200-219 74 74 74 148 148 148 F F F 11 220-239 74 74 74 147 147 147 F F F 12 240-259 74 74 74 148 148 148 F F F 13 260-279 74 74 74 148 148 148 F F F 14 280-299 70 70 70 135 135 135 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 71 71 71 139 139 139 F F F 17 340-359 75 75 75 149 149 149 F F F 18 360-379 75 75 75 149 149 149 F F F 19 380-399 74 74 74 148 148 148 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 73 73 73 144 144 144 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 75 75 75 149 149 149 F F F 25 500-519 75 75 75 149 149 149 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 73 73 73 144 144 144 F F F Max gen: 75, Max flush: 149 Begin loop 38 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.776634879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777348734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777378669Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487} {"msg":"Crucible 02f53a49-89aa-49d8-ad64-2a7209df88ac has session id: ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777403765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Upstairs opts: Upstairs UUID: 02f53a49-89aa-49d8-ad64-2a7209df88ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.77743972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Crucible stats registered with UUID: 02f53a49-89aa-49d8-ad64-2a7209df88ac","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777470456Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777683654Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487} {"msg":"02f53a49-89aa-49d8-ad64-2a7209df88ac active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777715889Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777740436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777768091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777791308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777814874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777838631Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777865577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777891963Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777917079Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.777941216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:38466)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778079075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:52.778119949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:52.778152304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:23:52.77818324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778214035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778241761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57468)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778267457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778295123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778759465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778798109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.778826935Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.77885469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.779413817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.779441583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.77993686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.779960287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.779974374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.780036015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.780053702Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:23:52.78006991Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.77885729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_connection connected from Ok(127.0.0.1:53938)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.77899218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"io task","client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.780117103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.780145809Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.780711725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781289769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781317575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[0]R flush_numbers[0..12]: [149, 117, 148, 144, 149, 137, 144, 146, 149, 141, 148, 147]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781334293Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[0]R generation[0..12]: [75, 62, 74, 73, 75, 70, 73, 74, 75, 72, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781348641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781362659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[1]R flush_numbers[0..12]: [149, 117, 148, 144, 149, 137, 144, 146, 149, 141, 148, 147]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781376637Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[1]R generation[0..12]: [75, 62, 74, 73, 75, 70, 73, 74, 75, 72, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781390364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781404422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[2]R flush_numbers[0..12]: [149, 117, 148, 144, 149, 137, 144, 146, 149, 141, 148, 147]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.7814224Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[2]R generation[0..12]: [75, 62, 74, 73, 75, 70, 73, 74, 75, 72, 74, 74]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781436268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781450355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:24:02.781463213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781478371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Next flush: 150","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781491429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781504227Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"":"downstairs","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781517465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781529304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"0","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781542742Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"1","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.78155576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"client":"2","":"downstairs","session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781569018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781581146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"02f53a49-89aa-49d8-ad64-2a7209df88ac is now active with session: ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781593804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781605543Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487,"session_id":"ce4d87e0-ee1e-4be6-bd03-1e3a2fdf6b8d"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:02.781617691Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 376 len 36864 data: 7 8 10 10 11 9 7 6 6 02/30 Write block 201 len 32768 data: 6 6 5 6 7 9 11 9 03/30 Write block 135 len 16384 data: 7 7 7 6 04/30 Read block 312 len 36864 05/30 Write block 181 len 28672 data: 6 5 6 5 6 6 6 06/30 Write block 276 len 12288 data: 9 8 7 07/30 Read block 61 len 28672 08/30 Read block 544 len 40960 09/30 Read block 72 len 24576 10/30 Write block 429 len 28672 data: 5 6 7 7 7 7 5 11/30 Flush 12/30 Write block 250 len 20480 data: 8 9 10 11 10 13/30 Write block 151 len 4096 data: 5 14/30 Read block 552 len 8192 15/30 Read block 482 len 4096 16/30 Write block 287 len 32768 data: 11 9 9 11 11 9 6 7 17/30 Flush 18/30 Write block 60 len 20480 data: 8 8 7 6 6 19/30 Read block 188 len 40960 20/30 Write block 95 len 4096 data: 7 21/30 Read block 84 len 20480 22/30 Write block 256 len 32768 data: 7 8 9 8 6 7 7 6 23/30 Read block 32 len 16384 24/30 Write block 133 len 20480 data: 7 7 8 8 8 25/30 Read block 413 len 4096 26/30 Write block 320 len 36864 data: 5 5 5 6 7 7 8 7 8 27/30 Write block 44 len 4096 data: 8 28/30 Read block 307 len 32768 29/30 Write block 142 len 32768 data: 8 9 10 10 8 6 6 6 30/30 Read block 498 len 20480 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 8 Done Done Sent false 30 Acked 1029 Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 2 3 Done 12 13 11 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:02.80786718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1487} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 149 149 149 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 74 74 74 148 148 148 T T T 3 060-079 73 73 73 144 144 144 T T T 4 080-099 75 75 75 149 149 149 T T T 5 100-119 70 70 70 137 137 137 F F F 6 120-139 76 76 76 150 150 150 T T T 7 140-159 76 76 76 151 151 151 T T T 8 160-179 75 75 75 149 149 149 F F F 9 180-199 76 76 76 150 150 150 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 74 74 74 147 147 147 F F F 12 240-259 76 76 76 151 151 151 T T T 13 260-279 76 76 76 150 150 150 T T T 14 280-299 76 76 76 151 151 151 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 71 71 71 139 139 139 T T T 17 340-359 75 75 75 149 149 149 F F F 18 360-379 76 76 76 150 150 150 F F F 19 380-399 76 76 76 150 150 150 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 76 76 76 150 150 150 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 75 75 75 149 149 149 F F F 25 500-519 75 75 75 149 149 149 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 73 73 73 144 144 144 F F F Max gen: 76, Max flush: 151 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 149 149 149 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 77 77 77 152 152 152 F F F 3 060-079 77 77 77 152 152 152 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 70 70 70 137 137 137 F F F 6 120-139 77 77 77 152 152 152 F F F 7 140-159 77 77 77 152 152 152 F F F 8 160-179 75 75 75 149 149 149 F F F 9 180-199 76 76 76 150 150 150 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 74 74 74 147 147 147 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 77 77 77 152 152 152 F F F 14 280-299 76 76 76 151 151 151 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 77 77 77 152 152 152 F F F 17 340-359 75 75 75 149 149 149 F F F 18 360-379 76 76 76 150 150 150 F F F 19 380-399 76 76 76 150 150 150 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 76 76 76 150 150 150 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 75 75 75 149 149 149 F F F 25 500-519 75 75 75 149 149 149 F F F 26 520-539 68 68 68 133 133 133 F F F 27 540-559 63 63 63 120 120 120 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 73 73 73 144 144 144 F F F Max gen: 77, Max flush: 152 Begin loop 39 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.056526518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057102402Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057132918Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495} {"msg":"Crucible 40f54375-3c8c-433b-9e2b-4200a313d91c has session id: 8e3fc999-53ff-42a2-b8c9-1a23f4179514","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057215446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Upstairs opts: Upstairs UUID: 40f54375-3c8c-433b-9e2b-4200a313d91c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.05732082Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Crucible stats registered with UUID: 40f54375-3c8c-433b-9e2b-4200a313d91c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057356685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057653471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495} {"msg":"40f54375-3c8c-433b-9e2b-4200a313d91c active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.05772601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057750866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057766744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057780692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.05779401Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057806898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057820816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057875038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057904233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.057983342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:17.058072899Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:17.058119362Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:17.058154056Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_connection connected from Ok(127.0.0.1:38666)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058180373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058210188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058237564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058301395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_connection connected from Ok(127.0.0.1:45399)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058431935Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058810949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058836555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058973075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.058992092Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.05941362Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.059557099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.059946111Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.059968578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.059985615Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.060065263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.06008383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:17.060097239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.059621929Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_connection connected from Ok(127.0.0.1:55908)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.059759588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"io task","client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.060695079Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.060729165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061295941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061861587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061889553Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[0]R flush_numbers[0..12]: [149, 117, 152, 152, 152, 137, 152, 152, 149, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.06190659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[0]R generation[0..12]: [75, 62, 77, 77, 77, 70, 77, 77, 75, 76, 76, 74]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061921438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061935836Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[1]R flush_numbers[0..12]: [149, 117, 152, 152, 152, 137, 152, 152, 149, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061949764Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[1]R generation[0..12]: [75, 62, 77, 77, 77, 70, 77, 77, 75, 76, 76, 74]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061967371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061981679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[2]R flush_numbers[0..12]: [149, 117, 152, 152, 152, 137, 152, 152, 149, 150, 150, 147]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.061995567Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[2]R generation[0..12]: [75, 62, 77, 77, 77, 70, 77, 77, 75, 76, 76, 74]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062009675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062023633Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Max found gen is 78","v":0,"name":"crucible","Wait for a query_work_queue command to finish before sending IO level":30,"time":"2024-08-05T19:24:27.06204018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062053518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Next flush: 153","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062066286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062079054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"":"downstairs","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062092403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062104461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"0","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062117879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"1","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062131057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"client":"2","":"downstairs","session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062144455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062156473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"40f54375-3c8c-433b-9e2b-4200a313d91c is now active with session: 8e3fc999-53ff-42a2-b8c9-1a23f4179514","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062171411Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062183389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495,"session_id":"8e3fc999-53ff-42a2-b8c9-1a23f4179514"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:27.062195487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 24576 data: 5 4 3 4 8 9 02/30 Read block 336 len 4096 03/30 Flush 04/30 Write block 8 len 16384 data: 7 9 10 9 05/30 Read block 408 len 28672 06/30 Write block 225 len 40960 data: 3 3 3 3 4 3 3 5 5 5 07/30 Write block 132 len 4096 data: 7 08/30 Write block 531 len 28672 data: 5 6 7 7 6 4 4 09/30 Read block 494 len 8192 10/30 Read block 554 len 8192 11/30 Write block 151 len 4096 data: 6 12/30 Read block 289 len 16384 13/30 Read block 184 len 20480 14/30 Write block 113 len 16384 data: 12 9 10 9 15/30 Write block 41 len 32768 data: 3 5 5 9 7 7 6 3 16/30 Write block 269 len 4096 data: 7 17/30 Write block 480 len 12288 data: 6 6 6 18/30 Write block 534 len 28672 data: 8 7 5 5 3 4 3 19/30 Read block 289 len 12288 20/30 Read block 350 len 28672 21/30 Read block 101 len 40960 22/30 Flush 23/30 Write block 584 len 8192 data: 4 5 24/30 Write block 228 len 16384 data: 4 5 4 4 25/30 Read block 93 len 20480 26/30 Flush 27/30 Write block 485 len 4096 data: 9 28/30 Read block 94 len 36864 29/30 Write block 153 len 4096 data: 6 30/30 Read block 206 len 4096 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 1 Done Done Done false 31 Acked 1030 Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:27.088076281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1495} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 154 154 154 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 78 78 78 154 154 154 F F F 3 060-079 77 77 77 152 152 152 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 78 78 78 154 154 154 F F F 7 140-159 78 78 78 154 154 154 T T T 8 160-179 75 75 75 149 149 149 F F F 9 180-199 76 76 76 150 150 150 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 78 78 78 156 156 156 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 78 78 78 154 154 154 F F F 14 280-299 76 76 76 151 151 151 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 77 77 77 152 152 152 F F F 17 340-359 75 75 75 149 149 149 F F F 18 360-379 76 76 76 150 150 150 F F F 19 380-399 76 76 76 150 150 150 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 76 76 76 150 150 150 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 78 78 78 154 154 154 T T T 25 500-519 78 78 78 153 153 153 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 78 78 78 154 154 154 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 78, Max flush: 156 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 154 154 154 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 78 78 78 154 154 154 F F F 3 060-079 77 77 77 152 152 152 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 78 78 78 154 154 154 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 75 75 75 149 149 149 F F F 9 180-199 76 76 76 150 150 150 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 78 78 78 156 156 156 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 78 78 78 154 154 154 F F F 14 280-299 76 76 76 151 151 151 F F F 15 300-319 72 72 72 143 143 143 F F F 16 320-339 77 77 77 152 152 152 F F F 17 340-359 75 75 75 149 149 149 F F F 18 360-379 76 76 76 150 150 150 F F F 19 380-399 76 76 76 150 150 150 F F F 20 400-419 70 70 70 137 137 137 F F F 21 420-439 76 76 76 150 150 150 F F F 22 440-459 70 70 70 137 137 137 F F F 23 460-479 72 72 72 143 143 143 F F F 24 480-499 79 79 79 157 157 157 F F F 25 500-519 78 78 78 153 153 153 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 78 78 78 154 154 154 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 79, Max flush: 157 Begin loop 40 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.270581155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271141442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271172558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503} {"msg":"Crucible 630909e9-2dbd-4258-a266-df714b4d79e3 has session id: 9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271379697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Upstairs opts: Upstairs UUID: 630909e9-2dbd-4258-a266-df714b4d79e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271423091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Crucible stats registered with UUID: 630909e9-2dbd-4258-a266-df714b4d79e3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271450587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271811643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503} {"msg":"630909e9-2dbd-4258-a266-df714b4d79e3 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271841658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271855906Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271871164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271884742Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.27189904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271912418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271925676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.27196977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.271988047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.272001965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:41.272284733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:41.272320678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_connection connected from Ok(127.0.0.1:56724)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.272355192Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:41.272385578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.272406865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.272430601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.272455747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_connection connected from Ok(127.0.0.1:39242)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.272588248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.273012015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.273038831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.273141236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.273161723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.273504072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.273612426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.274033414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.27405599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.274070058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.27412939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.274148137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:41.274161525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.273366634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_connection connected from Ok(127.0.0.1:39439)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.273510322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"io task","client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.274581784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.27460868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275154059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275728734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275763448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[0]R flush_numbers[0..12]: [154, 117, 154, 152, 152, 154, 154, 157, 149, 150, 150, 156]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275779976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[0]R generation[0..12]: [78, 62, 78, 77, 77, 78, 78, 79, 75, 76, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275796803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275811621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[1]R flush_numbers[0..12]: [154, 117, 154, 152, 152, 154, 154, 157, 149, 150, 150, 156]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275825309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[1]R generation[0..12]: [78, 62, 78, 77, 77, 78, 78, 79, 75, 76, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275838707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275857064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[2]R flush_numbers[0..12]: [154, 117, 154, 152, 152, 154, 154, 157, 149, 150, 150, 156]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275871172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[2]R generation[0..12]: [78, 62, 78, 77, 77, 78, 78, 79, 75, 76, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.27588473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275898708Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275911447Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275929114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Next flush: 158","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275942372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.27595549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"":"downstairs","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275968548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275980326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"0","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.275993754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"1","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.276006872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"client":"2","":"downstairs","session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.27601997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.276032098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"630909e9-2dbd-4258-a266-df714b4d79e3 is now active with session: 9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.276044797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.276059674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503,"session_id":"9fca7ffe-50f5-4f2a-b4e1-b79003a5fb06"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:24:51.276071763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 40960 data: 8 9 8 8 6 7 6 6 6 7 02/30 Flush 03/30 Write block 406 len 32768 data: 9 9 7 8 8 9 8 6 04/30 Write block 122 len 8192 data: 4 4 05/30 Write block 459 len 36864 data: 5 5 5 6 5 4 5 7 10 06/30 Read block 471 len 12288 07/30 Write block 495 len 16384 data: 5 6 8 6 08/30 Read block 109 len 8192 09/30 Read block 355 len 28672 10/30 Read block 541 len 16384 11/30 Read block 466 len 8192 12/30 Read block 420 len 32768 13/30 Write block 350 len 4096 data: 7 14/30 Read block 14 len 20480 15/30 Read block 575 len 40960 16/30 Read block 193 len 40960 17/30 Write block 548 len 12288 data: 4 4 4 18/30 Read block 441 len 32768 19/30 Write block 274 len 28672 data: 9 10 10 9 8 6 5 20/30 Write block 432 len 32768 data: 8 8 8 6 7 6 5 4 21/30 Write block 501 len 24576 data: 5 4 5 9 10 9 22/30 Read block 584 len 28672 23/30 Write block 125 len 28672 data: 3 4 6 5 8 8 7 24/30 Read block 590 len 36864 25/30 Write block 310 len 36864 data: 5 6 7 7 7 9 9 8 6 26/30 Read block 412 len 4096 27/30 Read block 447 len 32768 28/30 Write block 373 len 8192 data: 11 11 29/30 Read block 414 len 12288 30/30 Read block 95 len 24576 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 8 Done Done Done false 22 Acked 1021 Write 6 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Write 7 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 3 Done Done Done false 31 Acked 1030 Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 11 10 11 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:24:51.303797753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1503} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 154 154 154 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 78 78 78 154 154 154 F F F 3 060-079 77 77 77 152 152 152 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 80 80 80 159 159 159 T T T 7 140-159 79 79 79 157 157 157 F F F 8 160-179 80 80 80 158 158 158 F F F 9 180-199 80 80 80 158 158 158 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 78 78 78 156 156 156 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 80 80 80 159 159 159 F F F 14 280-299 80 80 80 159 159 159 F F F 15 300-319 72 72 72 143 143 143 T T T 16 320-339 77 77 77 152 152 152 F F F 17 340-359 80 80 80 159 159 159 F F F 18 360-379 76 76 76 150 150 150 T T T 19 380-399 76 76 76 150 150 150 F F F 20 400-419 80 80 80 159 159 159 F F F 21 420-439 76 76 76 150 150 150 T T T 22 440-459 80 80 80 159 159 159 F F F 23 460-479 80 80 80 159 159 159 F F F 24 480-499 80 80 80 159 159 159 F F F 25 500-519 78 78 78 153 153 153 T T T 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 80, Max flush: 159 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 154 154 154 F F F 1 020-039 62 62 62 117 117 117 F F F 2 040-059 78 78 78 154 154 154 F F F 3 060-079 77 77 77 152 152 152 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 81 81 81 160 160 160 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 80 80 80 158 158 158 F F F 9 180-199 80 80 80 158 158 158 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 78 78 78 156 156 156 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 80 80 80 159 159 159 F F F 14 280-299 80 80 80 159 159 159 F F F 15 300-319 81 81 81 160 160 160 F F F 16 320-339 77 77 77 152 152 152 F F F 17 340-359 80 80 80 159 159 159 F F F 18 360-379 81 81 81 160 160 160 F F F 19 380-399 76 76 76 150 150 150 F F F 20 400-419 80 80 80 159 159 159 F F F 21 420-439 81 81 81 160 160 160 F F F 22 440-459 80 80 80 159 159 159 F F F 23 460-479 80 80 80 159 159 159 F F F 24 480-499 80 80 80 159 159 159 F F F 25 500-519 81 81 81 160 160 160 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 75 75 75 149 149 149 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 81, Max flush: 160 Begin loop 41 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.519107053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.519639744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.51967065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511} {"msg":"Crucible 838d51db-c08d-42fe-9892-90db0b254804 has session id: d4df689f-ed58-473e-aa21-44ade4d781b5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.51980467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Upstairs opts: Upstairs UUID: 838d51db-c08d-42fe-9892-90db0b254804, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.519894657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Crucible stats registered with UUID: 838d51db-c08d-42fe-9892-90db0b254804","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.519913244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520321263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511} {"msg":"838d51db-c08d-42fe-9892-90db0b254804 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520416499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520435286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520455903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.52048076Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520507346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520530992Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520555779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520579735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520604012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520628348Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41888)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520790534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:49620)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.520922334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:05.521011261Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:05.521045986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:05.521064523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521077101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521090019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521193984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521728025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521753451Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521767789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.521781167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.522365031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.522487792Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.522883814Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.522915249Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.522931696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.522991618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.523010705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:05.523024233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.521909608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:32933)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.522045678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"io task","client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.522985639Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.523014064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.523697873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524292485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524318911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[0]R flush_numbers[0..12]: [154, 117, 154, 152, 152, 154, 160, 157, 158, 158, 150, 156]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524335019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[0]R generation[0..12]: [78, 62, 78, 77, 77, 78, 81, 79, 80, 80, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524349806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524364124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[1]R flush_numbers[0..12]: [154, 117, 154, 152, 152, 154, 160, 157, 158, 158, 150, 156]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524377972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[1]R generation[0..12]: [78, 62, 78, 77, 77, 78, 81, 79, 80, 80, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.52439581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524410617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[2]R flush_numbers[0..12]: [154, 117, 154, 152, 152, 154, 160, 157, 158, 158, 150, 156]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524424465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[2]R generation[0..12]: [78, 62, 78, 77, 77, 78, 81, 79, 80, 80, 76, 78]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524438013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2024-08-05T19:25:15.524455441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524468459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524484686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524506443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524520281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"":"downstairs","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524533389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524545177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"0","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524558335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"1","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524575903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"client":"2","":"downstairs","session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524599469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524625685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"838d51db-c08d-42fe-9892-90db0b254804 is now active with session: d4df689f-ed58-473e-aa21-44ade4d781b5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524649992Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524675028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511,"session_id":"d4df689f-ed58-473e-aa21-44ade4d781b5"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:15.524698335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 130 len 16384 data: 9 8 8 8 02/30 Write block 301 len 16384 data: 6 5 6 7 03/30 Read block 222 len 12288 04/30 Read block 200 len 4096 05/30 Write block 2 len 24576 data: 4 6 5 6 6 6 06/30 Read block 203 len 8192 07/30 Read block 277 len 12288 08/30 Read block 25 len 32768 09/30 Read block 458 len 20480 10/30 Flush 11/30 Write block 41 len 40960 data: 4 6 6 10 8 8 7 4 4 4 12/30 Read block 88 len 32768 13/30 Write block 163 len 8192 data: 11 11 14/30 Write block 305 len 28672 data: 7 5 4 3 4 6 7 15/30 Read block 12 len 40960 16/30 Write block 167 len 20480 data: 11 9 11 9 9 17/30 Read block 137 len 20480 18/30 Write block 399 len 4096 data: 6 19/30 Read block 31 len 36864 20/30 Write block 76 len 12288 data: 4 5 6 21/30 Write block 578 len 4096 data: 6 22/30 Read block 10 len 16384 23/30 Write block 426 len 8192 data: 6 7 24/30 Read block 503 len 12288 25/30 Write block 9 len 4096 data: 10 26/30 Write block 23 len 40960 data: 6 6 4 4 6 6 6 5 6 5 27/30 Write block 231 len 8192 data: 5 6 28/30 Read block 293 len 24576 29/30 Write block 479 len 8192 data: 6 7 30/30 Read block 415 len 8192 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 2 Done Done Done false 31 Acked 1030 Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 0 1 Done 7 8 8 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:15.54787095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1511} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 82 82 82 161 161 161 T T T 1 020-039 62 62 62 117 117 117 T T T 2 040-059 82 82 82 162 162 162 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 82 82 82 161 161 161 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 80 80 80 158 158 158 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 78 78 78 156 156 156 T T T 12 240-259 77 77 77 152 152 152 F F F 13 260-279 80 80 80 159 159 159 F F F 14 280-299 80 80 80 159 159 159 F F F 15 300-319 82 82 82 162 162 162 F F F 16 320-339 77 77 77 152 152 152 F F F 17 340-359 80 80 80 159 159 159 F F F 18 360-379 81 81 81 160 160 160 F F F 19 380-399 82 82 82 162 162 162 F F F 20 400-419 80 80 80 159 159 159 F F F 21 420-439 81 81 81 160 160 160 T T T 22 440-459 80 80 80 159 159 159 F F F 23 460-479 80 80 80 159 159 159 T T T 24 480-499 80 80 80 159 159 159 T T T 25 500-519 81 81 81 160 160 160 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 82 82 82 162 162 162 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 82, Max flush: 162 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 83 83 83 163 163 163 F F F 2 040-059 82 82 82 162 162 162 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 82 82 82 161 161 161 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 80 80 80 158 158 158 F F F 10 200-219 76 76 76 150 150 150 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 80 80 80 159 159 159 F F F 14 280-299 80 80 80 159 159 159 F F F 15 300-319 82 82 82 162 162 162 F F F 16 320-339 77 77 77 152 152 152 F F F 17 340-359 80 80 80 159 159 159 F F F 18 360-379 81 81 81 160 160 160 F F F 19 380-399 82 82 82 162 162 162 F F F 20 400-419 80 80 80 159 159 159 F F F 21 420-439 83 83 83 163 163 163 F F F 22 440-459 80 80 80 159 159 159 F F F 23 460-479 83 83 83 163 163 163 F F F 24 480-499 83 83 83 163 163 163 F F F 25 500-519 81 81 81 160 160 160 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 82 82 82 162 162 162 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 83, Max flush: 163 Begin loop 42 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.777991405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.778416532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.778452417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519} {"msg":"Crucible 1213446a-8f9b-4519-8d3f-5d1ce1ba473c has session id: b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.778758801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Upstairs opts: Upstairs UUID: 1213446a-8f9b-4519-8d3f-5d1ce1ba473c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.778801905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Crucible stats registered with UUID: 1213446a-8f9b-4519-8d3f-5d1ce1ba473c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.778825452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779127177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519} {"msg":"1213446a-8f9b-4519-8d3f-5d1ce1ba473c active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779216203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779234611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779249229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779269946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779296252Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779322108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779346794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779435971Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779454138Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779470996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:29.779503021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:29.779531897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:29.779559133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779582019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779604436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_connection connected from Ok(127.0.0.1:54316)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779628562Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779652509Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34131)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.779694383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.78059311Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.780632614Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.780662219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.780688675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781206139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781326131Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781652942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781678858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781692956Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781843304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781861391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:29.781874709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.780760312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43124)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.780823523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"io task","client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.781828504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.78185628Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.78252717Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783168725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.7831977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[0]R flush_numbers[0..12]: [163, 163, 162, 162, 152, 154, 161, 157, 162, 158, 150, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783214738Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[0]R generation[0..12]: [83, 83, 82, 82, 77, 78, 82, 79, 82, 80, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783229365Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783243813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[1]R flush_numbers[0..12]: [163, 163, 162, 162, 152, 154, 161, 157, 162, 158, 150, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.78326298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[1]R generation[0..12]: [83, 83, 82, 82, 77, 78, 82, 79, 82, 80, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783286187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783313283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[2]R flush_numbers[0..12]: [163, 163, 162, 162, 152, 154, 161, 157, 162, 158, 150, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.78333051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[2]R generation[0..12]: [83, 83, 82, 82, 77, 78, 82, 79, 82, 80, 76, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783344279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783358376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783375334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783388902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Next flush: 164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.78340164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783414468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"":"downstairs","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783427546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783439444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"0","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783452672Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"1","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783465641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"client":"2","":"downstairs","session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783480548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783507034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"1213446a-8f9b-4519-8d3f-5d1ce1ba473c is now active with session: b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783526312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.78353873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519,"session_id":"b236b316-a5fa-4b6d-a6ab-4c6855dfbaa0"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:39.783551218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 24576 data: 5 4 3 3 6 6 02/30 Write block 503 len 8192 data: 6 10 03/30 Flush 04/30 Write block 44 len 12288 data: 11 9 9 05/30 Read block 547 len 32768 06/30 Write block 388 len 36864 data: 8 10 11 10 9 9 10 9 10 07/30 Read block 489 len 16384 08/30 Read block 60 len 40960 09/30 Write block 428 len 24576 data: 6 6 7 8 9 9 10/30 Read block 451 len 24576 11/30 Write block 302 len 4096 data: 6 12/30 Flush 13/30 Write block 336 len 8192 data: 9 9 14/30 Read block 357 len 28672 15/30 Read block 444 len 36864 16/30 Write block 492 len 36864 data: 7 5 5 6 7 9 7 5 6 17/30 Write block 332 len 40960 data: 7 8 8 7 10 10 8 8 7 7 18/30 Read block 46 len 32768 19/30 Write block 482 len 28672 data: 7 7 9 10 8 9 12 20/30 Read block 281 len 12288 21/30 Read block 227 len 16384 22/30 Write block 310 len 4096 data: 7 23/30 Write block 426 len 24576 data: 7 8 7 7 8 9 24/30 Flush 25/30 Write block 123 len 24576 data: 5 3 4 5 7 6 26/30 Read block 110 len 16384 27/30 Read block 337 len 16384 28/30 Write block 473 len 16384 data: 4 4 5 6 29/30 Read block 282 len 40960 30/30 Write block 400 len 28672 data: 7 7 7 7 8 10 10 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:39.811988223Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1519} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 83 83 83 163 163 163 F F F 2 040-059 84 84 84 165 165 165 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 82 82 82 161 161 161 T T T 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 84 84 84 164 164 164 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 80 80 80 159 159 159 F F F 14 280-299 80 80 80 159 159 159 F F F 15 300-319 84 84 84 167 167 167 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 84 84 84 166 166 166 F F F 18 360-379 81 81 81 160 160 160 F F F 19 380-399 84 84 84 165 165 165 F F F 20 400-419 80 80 80 159 159 159 T T T 21 420-439 84 84 84 167 167 167 F F F 22 440-459 80 80 80 159 159 159 F F F 23 460-479 83 83 83 163 163 163 T T T 24 480-499 84 84 84 167 167 167 F F F 25 500-519 84 84 84 166 166 166 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 82 82 82 162 162 162 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 84, Max flush: 167 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 83 83 83 163 163 163 F F F 2 040-059 84 84 84 165 165 165 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 78 78 78 154 154 154 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 84 84 84 164 164 164 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 80 80 80 159 159 159 F F F 14 280-299 80 80 80 159 159 159 F F F 15 300-319 84 84 84 167 167 167 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 84 84 84 166 166 166 F F F 18 360-379 81 81 81 160 160 160 F F F 19 380-399 84 84 84 165 165 165 F F F 20 400-419 85 85 85 168 168 168 F F F 21 420-439 84 84 84 167 167 167 F F F 22 440-459 80 80 80 159 159 159 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 84 84 84 167 167 167 F F F 25 500-519 84 84 84 166 166 166 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 82 82 82 162 162 162 F F F 29 580-599 78 78 78 156 156 156 F F F Max gen: 85, Max flush: 168 Begin loop 43 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.004732109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005220276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.00526015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527} {"msg":"Crucible ada56359-3591-47a6-a930-20c477919f79 has session id: 048476ca-bf63-454e-bc13-e0c38918a4e6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005470659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Upstairs opts: Upstairs UUID: ada56359-3591-47a6-a930-20c477919f79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005518662Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Crucible stats registered with UUID: ada56359-3591-47a6-a930-20c477919f79","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005537419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.00579955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527} {"msg":"ada56359-3591-47a6-a930-20c477919f79 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005824057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005838065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005852933Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.00586678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005880629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005893776Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005906985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.00593461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005961286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.005986453Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:54.006150378Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:40592)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.006186413Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:64319)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.006211529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:54.006334281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:25:54.006372985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.006388703Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.006402581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.006513944Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.007008951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.007045536Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.007073762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.007102247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.007629259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.007656855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.00816539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.008206884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.008221771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.008247508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.008261916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:25:54.008275243Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.007442117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:44650)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.00755809Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"io task","client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.0087021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.008741014Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009285724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.00985069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009878756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[0]R flush_numbers[0..12]: [163, 163, 165, 162, 152, 154, 168, 157, 162, 164, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009895554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[0]R generation[0..12]: [83, 83, 84, 82, 77, 78, 85, 79, 82, 84, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009910721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009925429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[1]R flush_numbers[0..12]: [163, 163, 165, 162, 152, 154, 168, 157, 162, 164, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009943426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[1]R generation[0..12]: [83, 83, 84, 82, 77, 78, 85, 79, 82, 84, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009957554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009971642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[2]R flush_numbers[0..12]: [163, 163, 165, 162, 152, 154, 168, 157, 162, 164, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.00998534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[2]R generation[0..12]: [83, 83, 84, 82, 77, 78, 85, 79, 82, 84, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.009998968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010013176Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010030234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010043472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Next flush: 169","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.01005629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010069118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"":"downstairs","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010082466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010094694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"0","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010108092Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"1","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.01012123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"client":"2","":"downstairs","session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010136758Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010149206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"ada56359-3591-47a6-a930-20c477919f79 is now active with session: 048476ca-bf63-454e-bc13-e0c38918a4e6","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010161914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.010173772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527,"session_id":"048476ca-bf63-454e-bc13-e0c38918a4e6"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:04.0101859Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 589 len 8192 data: 5 7 02/30 Read block 485 len 4096 03/30 Write block 344 len 12288 data: 9 11 11 04/30 Write block 277 len 28672 data: 10 9 7 6 4 5 7 05/30 Write block 507 len 36864 data: 9 9 9 8 8 8 5 8 7 06/30 Read block 72 len 4096 07/30 Read block 491 len 32768 08/30 Flush 09/30 Write block 580 len 12288 data: 8 8 6 10/30 Write block 368 len 40960 data: 10 9 8 8 7 12 12 11 8 9 11/30 Read block 433 len 28672 12/30 Read block 389 len 32768 13/30 Write block 109 len 24576 data: 7 7 9 12 13 10 14/30 Read block 322 len 32768 15/30 Read block 255 len 4096 16/30 Read block 303 len 40960 17/30 Flush 18/30 Write block 366 len 24576 data: 10 12 11 10 9 9 19/30 Write block 32 len 8192 data: 6 6 20/30 Write block 495 len 36864 data: 7 8 10 8 6 7 6 5 7 21/30 Read block 435 len 40960 22/30 Read block 379 len 28672 23/30 Read block 185 len 32768 24/30 Write block 490 len 16384 data: 10 8 8 6 25/30 Write block 47 len 32768 data: 8 5 5 5 5 6 7 6 26/30 Read block 23 len 16384 27/30 Read block 21 len 36864 28/30 Read block 256 len 16384 29/30 Read block 258 len 20480 30/30 Write block 397 len 32768 data: 7 7 7 8 8 8 8 9 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Write 9 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:04.036180777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1527} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 83 83 83 163 163 163 T T T 2 040-059 84 84 84 165 165 165 T T T 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 86 86 86 170 170 170 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 84 84 84 164 164 164 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 86 86 86 169 169 169 F F F 14 280-299 86 86 86 169 169 169 F F F 15 300-319 84 84 84 167 167 167 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 86 86 86 169 169 169 F F F 18 360-379 86 86 86 170 170 170 T T T 19 380-399 84 84 84 165 165 165 T T T 20 400-419 85 85 85 168 168 168 T T T 21 420-439 84 84 84 167 167 167 F F F 22 440-459 80 80 80 159 159 159 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 84 84 84 167 167 167 T T T 25 500-519 86 86 86 169 169 169 T T T 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 82 82 82 162 162 162 F F F 29 580-599 86 86 86 170 170 170 F F F Max gen: 86, Max flush: 170 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 87 87 87 171 171 171 F F F 2 040-059 87 87 87 171 171 171 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 86 86 86 170 170 170 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 84 84 84 164 164 164 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 86 86 86 169 169 169 F F F 14 280-299 86 86 86 169 169 169 F F F 15 300-319 84 84 84 167 167 167 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 86 86 86 169 169 169 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 87 87 87 171 171 171 F F F 20 400-419 87 87 87 171 171 171 F F F 21 420-439 84 84 84 167 167 167 F F F 22 440-459 80 80 80 159 159 159 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 87 87 87 171 171 171 F F F 25 500-519 87 87 87 171 171 171 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 80 80 80 159 159 159 F F F 28 560-579 82 82 82 162 162 162 F F F 29 580-599 86 86 86 170 170 170 F F F Max gen: 87, Max flush: 171 Begin loop 44 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.280595892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.280979075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281008121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535} {"msg":"Crucible 21e4308f-bf90-4194-95ae-41b6d368b3a0 has session id: 5e12d149-4278-4d54-a466-0ad3c9bbc23e","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281267572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Upstairs opts: Upstairs UUID: 21e4308f-bf90-4194-95ae-41b6d368b3a0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281310736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Crucible stats registered with UUID: 21e4308f-bf90-4194-95ae-41b6d368b3a0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281335022Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281632538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535} {"msg":"21e4308f-bf90-4194-95ae-41b6d368b3a0 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.28175652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281775697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281790835Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281804572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281818341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281837128Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281861344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281885391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281914796Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.281934743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:18.28215891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:18.282286011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34753)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282314557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:18.282342973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282369069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282394415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34474)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282418452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282433219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282947863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.28297049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282984598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.282998196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.283600137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.283630342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.28418176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.284206347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.284220385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.284280716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.284298923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:18.284312281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.282746963Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50830)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.282868965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"io task","client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.283874696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.283902221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.284565813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285120051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285146497Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[0]R flush_numbers[0..12]: [163, 171, 171, 162, 152, 170, 168, 157, 162, 164, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285162535Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[0]R generation[0..12]: [83, 87, 87, 82, 77, 86, 85, 79, 82, 84, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285181772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.28519651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[1]R flush_numbers[0..12]: [163, 171, 171, 162, 152, 170, 168, 157, 162, 164, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285210518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[1]R generation[0..12]: [83, 87, 87, 82, 77, 86, 85, 79, 82, 84, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285224205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285238354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[2]R flush_numbers[0..12]: [163, 171, 171, 162, 152, 170, 168, 157, 162, 164, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285252032Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[2]R generation[0..12]: [83, 87, 87, 82, 77, 86, 85, 79, 82, 84, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285265739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285279727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:26:28.285292566Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285305924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285318842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.28533165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"":"downstairs","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285344688Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285356386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"0","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285371404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"1","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285384732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"client":"2","":"downstairs","session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.28539794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285409998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"21e4308f-bf90-4194-95ae-41b6d368b3a0 is now active with session: 5e12d149-4278-4d54-a466-0ad3c9bbc23e","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285422686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285434564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535,"session_id":"5e12d149-4278-4d54-a466-0ad3c9bbc23e"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:28.285446653Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 311 len 4096 data: 8 02/30 Read block 8 len 12288 03/30 Write block 506 len 4096 data: 10 04/30 Read block 78 len 28672 05/30 Flush 06/30 Write block 494 len 40960 data: 6 8 9 11 9 7 8 7 6 8 07/30 Write block 553 len 32768 data: 6 7 6 7 5 5 5 6 08/30 Read block 359 len 32768 09/30 Flush 10/30 Write block 502 len 36864 data: 7 9 11 11 11 10 10 10 9 11/30 Flush 12/30 Write block 436 len 16384 data: 8 7 6 5 13/30 Read block 397 len 12288 14/30 Read block 279 len 24576 15/30 Read block 251 len 4096 16/30 Read block 449 len 20480 17/30 Write block 26 len 32768 data: 5 7 7 7 6 7 7 7 18/30 Write block 384 len 12288 data: 7 5 6 19/30 Write block 108 len 8192 data: 7 8 20/30 Write block 197 len 4096 data: 4 21/30 Write block 455 len 4096 data: 6 22/30 Read block 557 len 16384 23/30 Write block 280 len 8192 data: 7 5 24/30 Write block 489 len 4096 data: 12 25/30 Read block 231 len 28672 26/30 Read block 123 len 16384 27/30 Write block 515 len 12288 data: 8 7 7 28/30 Read block 538 len 16384 29/30 Write block 511 len 32768 data: 9 9 6 9 9 8 8 8 30/30 Read block 513 len 24576 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 3 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 8 Done Done Done false 31 Acked 1030 Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:28.310297819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1535} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 88 88 88 175 175 175 F F F 2 040-059 87 87 87 171 171 171 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 86 86 86 170 170 170 T T T 6 120-139 85 85 85 168 168 168 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 84 84 84 164 164 164 T T T 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 86 86 86 169 169 169 F F F 14 280-299 86 86 86 169 169 169 T T T 15 300-319 88 88 88 172 172 172 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 86 86 86 169 169 169 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 87 87 87 171 171 171 T T T 20 400-419 87 87 87 171 171 171 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 80 80 80 159 159 159 T T T 23 460-479 85 85 85 168 168 168 F F F 24 480-499 88 88 88 173 173 173 T T T 25 500-519 88 88 88 174 174 174 T T T 26 520-539 78 78 78 154 154 154 F F F 27 540-559 88 88 88 173 173 173 F F F 28 560-579 88 88 88 173 173 173 F F F 29 580-599 86 86 86 170 170 170 F F F Max gen: 88, Max flush: 175 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 F F F 1 020-039 88 88 88 175 175 175 F F F 2 040-059 87 87 87 171 171 171 F F F 3 060-079 82 82 82 162 162 162 F F F 4 080-099 77 77 77 152 152 152 F F F 5 100-119 89 89 89 176 176 176 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 79 79 79 157 157 157 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 89 89 89 176 176 176 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 77 77 77 152 152 152 F F F 13 260-279 86 86 86 169 169 169 F F F 14 280-299 89 89 89 176 176 176 F F F 15 300-319 88 88 88 172 172 172 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 86 86 86 169 169 169 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 89 89 89 176 176 176 F F F 20 400-419 87 87 87 171 171 171 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 89 89 89 176 176 176 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 89 89 89 176 176 176 F F F 25 500-519 89 89 89 176 176 176 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 88 88 88 173 173 173 F F F 28 560-579 88 88 88 173 173 173 F F F 29 580-599 86 86 86 170 170 170 F F F Max gen: 89, Max flush: 176 Begin loop 45 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.557595599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558077558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558106353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543} {"msg":"Crucible d21251b7-93ab-41bb-9e15-9598a356e3e5 has session id: 2862beb5-7028-48d0-84db-90731add4092","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558144338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Upstairs opts: Upstairs UUID: d21251b7-93ab-41bb-9e15-9598a356e3e5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558292736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Crucible stats registered with UUID: d21251b7-93ab-41bb-9e15-9598a356e3e5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558320922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558687877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543} {"msg":"d21251b7-93ab-41bb-9e15-9598a356e3e5 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558783843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558807329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558833646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558859522Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558883088Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558907565Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558932041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558959367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.558985053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.55900783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:42.559096227Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_connection connected from Ok(127.0.0.1:56726)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.559134721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:42.559164816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_connection connected from Ok(127.0.0.1:52525)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.559191653Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:42.559218739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.559239136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.559261372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.559483699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.559996723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.560035278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.560065723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.560096258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.56069288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.560730295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.561258666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.561285822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.5613014Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.561316398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.561330495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:42.561343844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.559745741Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_connection connected from Ok(127.0.0.1:61439)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.559881591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"io task","client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.560880043Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.560908959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.561477625Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562099472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562122449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[0]R flush_numbers[0..12]: [163, 175, 171, 162, 152, 176, 168, 157, 162, 176, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562142296Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[0]R generation[0..12]: [83, 88, 87, 82, 77, 89, 85, 79, 82, 89, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562156744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562170872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[1]R flush_numbers[0..12]: [163, 175, 171, 162, 152, 176, 168, 157, 162, 176, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.56218468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[1]R generation[0..12]: [83, 88, 87, 82, 77, 89, 85, 79, 82, 89, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562198448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562212376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[2]R flush_numbers[0..12]: [163, 175, 171, 162, 152, 176, 168, 157, 162, 176, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562235072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[2]R generation[0..12]: [83, 88, 87, 82, 77, 89, 85, 79, 82, 89, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.56224932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562263408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562276306Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"Wait for a query_work_queue command to finish before sending IO msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562293024Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Next flush: 177","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562306102Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562320629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"":"downstairs","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562339627Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562362873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"0","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.56238249Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"1","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562406957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"client":"2","":"downstairs","session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562428053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.56244951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"d21251b7-93ab-41bb-9e15-9598a356e3e5 is now active with session: 2862beb5-7028-48d0-84db-90731add4092","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562473787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562493914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543,"session_id":"2862beb5-7028-48d0-84db-90731add4092"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:26:52.562511561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 83 len 12288 data: 6 6 5 02/30 Write block 480 len 40960 data: 8 7 8 8 10 11 9 10 13 13 03/30 Read block 324 len 24576 04/30 Flush 05/30 Write block 60 len 40960 data: 9 9 8 7 7 6 5 5 3 6 06/30 Read block 253 len 8192 07/30 Read block 523 len 16384 08/30 Write block 347 len 40960 data: 7 7 7 8 6 7 6 6 6 4 09/30 Read block 153 len 8192 10/30 Write block 556 len 4096 data: 8 11/30 Read block 5 len 8192 12/30 Read block 297 len 12288 13/30 Read block 276 len 24576 14/30 Read block 420 len 24576 15/30 Read block 101 len 16384 16/30 Write block 111 len 12288 data: 10 13 14 17/30 Write block 388 len 12288 data: 9 11 12 18/30 Flush 19/30 Write block 257 len 8192 data: 9 10 20/30 Read block 586 len 16384 21/30 Write block 146 len 20480 data: 9 7 7 7 6 22/30 Write block 271 len 4096 data: 7 23/30 Read block 475 len 12288 24/30 Write block 575 len 8192 data: 5 5 25/30 Write block 58 len 4096 data: 7 26/30 Write block 581 len 4096 data: 9 27/30 Read block 470 len 40960 28/30 Read block 397 len 32768 29/30 Write block 573 len 36864 data: 6 5 6 6 5 7 6 9 10 30/30 Write block 11 len 28672 data: 10 9 6 5 7 7 6 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:1 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 4 Done Done Done false 21 Acked 1020 Write 5 Done Done Done false 22 Acked 1021 Write 1 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 9 Sent Done Sent false 30 NotAcked 1029 Flush 0 Sent Sent Sent false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 3 2 3 8 Done 10 11 10 31 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 31 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:26:52.58498187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1543} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 83 83 83 163 163 163 T T T 1 020-039 88 88 88 175 175 175 F F F 2 040-059 90 90 90 179 179 179 F F F 3 060-079 90 90 90 178 178 178 F F F 4 080-099 90 90 90 177 177 177 F F F 5 100-119 90 90 90 178 178 178 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 90 90 90 179 179 179 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 89 89 89 176 176 176 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 90 90 90 179 179 179 F F F 13 260-279 90 90 90 179 179 179 F F F 14 280-299 89 89 89 176 176 176 F F F 15 300-319 88 88 88 172 172 172 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 90 90 90 178 178 178 F F F 20 400-419 87 87 87 171 171 171 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 89 89 89 176 176 176 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 90 90 90 177 177 177 F F F 25 500-519 89 89 89 176 176 176 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 90 90 90 178 178 178 F F F 28 560-579 90 90 90 179 179 179 F F F 29 580-599 90 90 90 179 179 179 F F F Max gen: 90, Max flush: 179 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 91 91 91 180 180 180 F F F 1 020-039 88 88 88 175 175 175 F F F 2 040-059 90 90 90 179 179 179 F F F 3 060-079 90 90 90 178 178 178 F F F 4 080-099 90 90 90 177 177 177 F F F 5 100-119 90 90 90 178 178 178 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 90 90 90 179 179 179 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 89 89 89 176 176 176 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 83 83 83 163 163 163 F F F 12 240-259 90 90 90 179 179 179 F F F 13 260-279 90 90 90 179 179 179 F F F 14 280-299 89 89 89 176 176 176 F F F 15 300-319 88 88 88 172 172 172 F F F 16 320-339 84 84 84 166 166 166 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 90 90 90 178 178 178 F F F 20 400-419 87 87 87 171 171 171 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 89 89 89 176 176 176 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 90 90 90 177 177 177 F F F 25 500-519 89 89 89 176 176 176 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 90 90 90 178 178 178 F F F 28 560-579 90 90 90 179 179 179 F F F 29 580-599 90 90 90 179 179 179 F F F Max gen: 91, Max flush: 180 Begin loop 46 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.756380954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.756904346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.756936352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551} {"msg":"Crucible 95e12cd7-aee6-4105-8f01-72ed86997988 has session id: e0165dcf-4406-4bac-8cbe-8572f66cc3f3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757087539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Upstairs opts: Upstairs UUID: 95e12cd7-aee6-4105-8f01-72ed86997988, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757126464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Crucible stats registered with UUID: 95e12cd7-aee6-4105-8f01-72ed86997988","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.75714952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757430069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551} {"msg":"95e12cd7-aee6-4105-8f01-72ed86997988 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757588985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757606262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.75762066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757634058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757647556Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757664714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757687561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757713297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757727775Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.75775524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:02.757931634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42340)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.757969049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:02.757999334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33546)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758024011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:02.758064115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758089761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758114327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758284492Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758750373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758777099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758795616Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.758809664Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.759460847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.759490283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.760101173Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.760127139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.760142157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.760154585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.760167863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:02.760180611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.759263527Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48865)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.759319239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"io task","client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.760282766Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.760314701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.760833894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761372395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.76140013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[0]R flush_numbers[0..12]: [180, 175, 179, 178, 177, 178, 168, 179, 162, 176, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761416928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[0]R generation[0..12]: [91, 88, 90, 90, 90, 90, 85, 90, 82, 89, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761431236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761445824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[1]R flush_numbers[0..12]: [180, 175, 179, 178, 177, 178, 168, 179, 162, 176, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761459551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[1]R generation[0..12]: [91, 88, 90, 90, 90, 90, 85, 90, 82, 89, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.76147304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761487078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[2]R flush_numbers[0..12]: [180, 175, 179, 178, 177, 178, 168, 179, 162, 176, 164, 163]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761500825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[2]R generation[0..12]: [91, 88, 90, 90, 90, 90, 85, 90, 82, 89, 84, 83]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761514383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:27:12.761528321Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761543529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761556857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761569475Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761585553Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"":"downstairs","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761598811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761610699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"0","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761623837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"1","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761636655Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"client":"2","":"downstairs","session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761649683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761661812Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"95e12cd7-aee6-4105-8f01-72ed86997988 is now active with session: e0165dcf-4406-4bac-8cbe-8572f66cc3f3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.7616746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761686478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551,"session_id":"e0165dcf-4406-4bac-8cbe-8572f66cc3f3"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:12.761698646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 307 len 24576 data: 5 4 5 8 9 8 02/30 Read block 187 len 36864 03/30 Read block 18 len 16384 04/30 Read block 564 len 40960 05/30 Write block 396 len 32768 data: 11 8 8 8 9 9 9 9 06/30 Read block 358 len 12288 07/30 Write block 420 len 12288 data: 5 4 4 08/30 Write block 589 len 40960 data: 6 8 7 7 7 7 8 6 5 3 09/30 Read block 532 len 32768 10/30 Write block 10 len 8192 data: 11 11 11/30 Write block 64 len 24576 data: 8 7 6 6 4 7 12/30 Read block 160 len 12288 13/30 Read block 353 len 4096 14/30 Read block 266 len 28672 15/30 Write block 330 len 4096 data: 7 16/30 Write block 221 len 12288 data: 8 5 4 17/30 Read block 410 len 36864 18/30 Read block 373 len 20480 19/30 Write block 555 len 40960 data: 7 9 6 6 6 7 8 8 10 9 20/30 Read block 440 len 4096 21/30 Write block 434 len 28672 data: 9 7 9 8 7 6 5 22/30 Write block 76 len 16384 data: 5 6 7 6 23/30 Write block 571 len 12288 data: 9 8 7 24/30 Read block 149 len 4096 25/30 Write block 51 len 28672 data: 6 7 8 7 8 8 8 26/30 Read block 280 len 8192 27/30 Write block 101 len 40960 data: 4 4 4 5 6 8 7 8 9 8 28/30 Write block 571 len 8192 data: 10 9 29/30 Write block 78 len 24576 data: 8 7 6 5 6 7 30/30 Write block 277 len 24576 data: 11 10 8 8 6 6 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:1 downstairs:31 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 6 Done Done Done false 2 Acked 1001 Read 9 Done Done Done false 3 Acked 1002 Read 4 Done Done Done false 4 Acked 1003 Read 10 Done Done Done false 5 Acked 1004 Write 8 Done Done Done false 6 Acked 1005 Read 3 Done Done Done false 7 Acked 1006 Write 3 Done Done Done false 8 Acked 1007 Write 10 Done Done Done false 9 Acked 1008 Read 8 Done Done Done false 10 Acked 1009 Write 2 Done Done Done false 11 Acked 1010 Write 6 Done Done Done false 12 Acked 1011 Read 3 Done Done Done false 13 Acked 1012 Read 1 Done Done Done false 14 Acked 1013 Read 7 Done Done Done false 15 Acked 1014 Write 1 Done Done Done false 16 Acked 1015 Write 3 Done Done Done false 17 Acked 1016 Read 9 Done Done Done false 18 Acked 1017 Read 5 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 1 Done Done Done false 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 4 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 NotAcked 1028 Flush 0 Sent Sent Sent false 30 Acked 1029 Write 6 Sent Sent Sent false 31 Acked 1030 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 3 3 3 9 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 31 30 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:12.783104683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1551} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 181 181 181 F F F 1 020-039 88 88 88 175 175 175 F F F 2 040-059 92 92 92 181 181 181 F F F 3 060-079 92 92 92 181 181 181 T T T 4 080-099 90 90 90 177 177 177 T T T 5 100-119 92 92 92 181 181 181 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 90 90 90 179 179 179 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 89 89 89 176 176 176 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 90 90 90 179 179 179 F F F 13 260-279 90 90 90 179 179 179 T T T 14 280-299 89 89 89 176 176 176 T T T 15 300-319 92 92 92 181 181 181 F F F 16 320-339 92 92 92 181 181 181 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 92 92 92 181 181 181 F F F 20 400-419 92 92 92 181 181 181 F F F 21 420-439 92 92 92 181 181 181 F F F 22 440-459 92 92 92 181 181 181 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 90 90 90 177 177 177 F F F 25 500-519 89 89 89 176 176 176 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 92 92 92 181 181 181 F F F 28 560-579 92 92 92 181 181 181 F F F 29 580-599 92 92 92 181 181 181 F F F Max gen: 92, Max flush: 181 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 181 181 181 F F F 1 020-039 88 88 88 175 175 175 F F F 2 040-059 92 92 92 181 181 181 F F F 3 060-079 93 93 93 182 182 182 F F F 4 080-099 93 93 93 182 182 182 F F F 5 100-119 92 92 92 181 181 181 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 90 90 90 179 179 179 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 89 89 89 176 176 176 F F F 10 200-219 84 84 84 164 164 164 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 90 90 90 179 179 179 F F F 13 260-279 93 93 93 182 182 182 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 92 92 92 181 181 181 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 87 87 87 171 171 171 F F F 19 380-399 92 92 92 181 181 181 F F F 20 400-419 92 92 92 181 181 181 F F F 21 420-439 92 92 92 181 181 181 F F F 22 440-459 92 92 92 181 181 181 F F F 23 460-479 85 85 85 168 168 168 F F F 24 480-499 90 90 90 177 177 177 F F F 25 500-519 89 89 89 176 176 176 F F F 26 520-539 78 78 78 154 154 154 F F F 27 540-559 92 92 92 181 181 181 F F F 28 560-579 92 92 92 181 181 181 F F F 29 580-599 92 92 92 181 181 181 F F F Max gen: 93, Max flush: 182 Begin loop 47 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.990788113Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.99155322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991585515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559} {"msg":"Crucible d586a4a1-184f-4610-96a3-b63c56eba354 has session id: fe97acca-1099-4b8e-8909-89ec2944af43","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991607242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Upstairs opts: Upstairs UUID: d586a4a1-184f-4610-96a3-b63c56eba354, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991640327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Crucible stats registered with UUID: d586a4a1-184f-4610-96a3-b63c56eba354","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991666313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991928764Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559} {"msg":"d586a4a1-184f-4610-96a3-b63c56eba354 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991954641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991970198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.991985326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992016582Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992031959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992047557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992070763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.99209432Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992119676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992142193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_connection connected from Ok(127.0.0.1:61776)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992289101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:26.992330505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:26.99236255Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:26.992411783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_connection connected from Ok(127.0.0.1:51398)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992486182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992514448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992539324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.992666385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.993197826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.993225022Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.99323956Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.993252948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.993856629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.993993528Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.994345326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.99438414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.994411726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.994473857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.994503893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:26.9945198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.994476305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_connection connected from Ok(127.0.0.1:38846)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.99464802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"io task","client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.995645042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.995671698Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996209528Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996772125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996790742Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[0]R flush_numbers[0..12]: [181, 175, 181, 182, 182, 181, 168, 179, 162, 176, 164, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.99680489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[0]R generation[0..12]: [92, 88, 92, 93, 93, 92, 85, 90, 82, 89, 84, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996818988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996833236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[1]R flush_numbers[0..12]: [181, 175, 181, 182, 182, 181, 168, 179, 162, 176, 164, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996846954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[1]R generation[0..12]: [92, 88, 92, 93, 93, 92, 85, 90, 82, 89, 84, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996860762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.99687468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[2]R flush_numbers[0..12]: [181, 175, 181, 182, 182, 181, 168, 179, 162, 176, 164, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996892727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[2]R generation[0..12]: [92, 88, 92, 93, 93, 92, 85, 90, 82, 89, 84, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996906895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996921143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996934181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996947109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996959837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"All extents match","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-08-05T19:27:36.996976134Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"":"downstairs","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.996989303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.99700888Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"0","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997027277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"1","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997040815Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"client":"2","":"downstairs","session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997053793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997066021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"d586a4a1-184f-4610-96a3-b63c56eba354 is now active with session: fe97acca-1099-4b8e-8909-89ec2944af43","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997078769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997090448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559,"session_id":"fe97acca-1099-4b8e-8909-89ec2944af43"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:36.997105355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 195 len 24576 data: 6 5 5 4 7 7 02/30 Write block 105 len 8192 data: 7 9 03/30 Flush 04/30 Write block 21 len 36864 data: 5 6 7 7 5 6 8 8 8 05/30 Flush 06/30 Write block 478 len 36864 data: 6 7 9 8 9 9 11 12 10 07/30 Read block 581 len 24576 08/30 Read block 156 len 20480 09/30 Write block 258 len 8192 data: 11 9 10/30 Read block 330 len 8192 11/30 Write block 523 len 40960 data: 7 7 5 4 5 4 4 4 6 7 12/30 Read block 370 len 40960 13/30 Read block 381 len 28672 14/30 Read block 12 len 40960 15/30 Write block 496 len 32768 data: 10 12 10 8 9 8 8 10 16/30 Flush 17/30 Write block 202 len 40960 data: 7 6 7 8 10 12 10 9 9 11 18/30 Read block 238 len 28672 19/30 Write block 42 len 32768 data: 7 7 12 10 10 9 6 6 20/30 Write block 573 len 8192 data: 8 6 21/30 Write block 564 len 24576 data: 10 9 7 7 9 7 22/30 Write block 443 len 20480 data: 8 6 6 6 5 23/30 Read block 443 len 8192 24/30 Write block 65 len 20480 data: 8 7 7 5 8 25/30 Write block 406 len 4096 data: 11 26/30 Write block 423 len 12288 data: 6 6 7 27/30 Write block 379 len 16384 data: 11 12 10 8 28/30 Read block 189 len 32768 29/30 Read block 70 len 40960 30/30 Write block 47 len 36864 data: 10 7 7 6 7 8 9 8 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 10 Done Done Done false 18 Acked 1017 Read 7 Done Done Done false 19 Acked 1018 Write 8 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:37.023877457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1559} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 181 181 181 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 92 92 92 181 181 181 T T T 3 060-079 93 93 93 182 182 182 T T T 4 080-099 93 93 93 182 182 182 F F F 5 100-119 94 94 94 183 183 183 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 90 90 90 179 179 179 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 94 94 94 183 183 183 F F F 10 200-219 94 94 94 183 183 183 T T T 11 220-239 92 92 92 181 181 181 F F F 12 240-259 94 94 94 185 185 185 F F F 13 260-279 93 93 93 182 182 182 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 92 92 92 181 181 181 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 87 87 87 171 171 171 T T T 19 380-399 92 92 92 181 181 181 T T T 20 400-419 92 92 92 181 181 181 T T T 21 420-439 92 92 92 181 181 181 T T T 22 440-459 92 92 92 181 181 181 T T T 23 460-479 94 94 94 185 185 185 F F F 24 480-499 94 94 94 185 185 185 F F F 25 500-519 94 94 94 185 185 185 F F F 26 520-539 94 94 94 185 185 185 F F F 27 540-559 92 92 92 181 181 181 F F F 28 560-579 92 92 92 181 181 181 T T T 29 580-599 92 92 92 181 181 181 F F F Max gen: 94, Max flush: 185 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 92 92 92 181 181 181 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 95 95 95 186 186 186 F F F 3 060-079 95 95 95 186 186 186 F F F 4 080-099 93 93 93 182 182 182 F F F 5 100-119 94 94 94 183 183 183 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 90 90 90 179 179 179 F F F 8 160-179 82 82 82 162 162 162 F F F 9 180-199 94 94 94 183 183 183 F F F 10 200-219 95 95 95 186 186 186 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 94 94 94 185 185 185 F F F 13 260-279 93 93 93 182 182 182 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 92 92 92 181 181 181 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 95 95 95 186 186 186 F F F 19 380-399 95 95 95 186 186 186 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 95 95 95 186 186 186 F F F 22 440-459 95 95 95 186 186 186 F F F 23 460-479 94 94 94 185 185 185 F F F 24 480-499 94 94 94 185 185 185 F F F 25 500-519 94 94 94 185 185 185 F F F 26 520-539 94 94 94 185 185 185 F F F 27 540-559 92 92 92 181 181 181 F F F 28 560-579 95 95 95 186 186 186 F F F 29 580-599 92 92 92 181 181 181 F F F Max gen: 95, Max flush: 186 Begin loop 48 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.293363407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.293761308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.293782955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567} {"msg":"Crucible b74ea671-7523-4db5-ae9e-bfb305f291a8 has session id: 5b2addd1-11b4-4baf-b996-c54168ac4781","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.293966988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Upstairs opts: Upstairs UUID: b74ea671-7523-4db5-ae9e-bfb305f291a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294060734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Crucible stats registered with UUID: b74ea671-7523-4db5-ae9e-bfb305f291a8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294091449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294417881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567} {"msg":"b74ea671-7523-4db5-ae9e-bfb305f291a8 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294442457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294456745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294472153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294485951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294499429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294512687Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294526045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294539133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294552731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294618021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:51.294773978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:51.294816542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_connection connected from Ok(127.0.0.1:33775)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294842298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_connection connected from Ok(127.0.0.1:42479)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.294871054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:27:51.294981797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295015512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295039169Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295141734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295662316Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295696681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295723757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.295750003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.29624573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.296383269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.296743986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.296779921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.296798408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.29691867Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.296937767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:27:51.296951025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.296411055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_connection connected from Ok(127.0.0.1:34393)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.296537906Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"io task","client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.297587831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.297615667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298046013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298574035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298602191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[0]R flush_numbers[0..12]: [181, 184, 186, 186, 182, 183, 168, 179, 162, 183, 186, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298618298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[0]R generation[0..12]: [92, 94, 95, 95, 93, 94, 85, 90, 82, 94, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298632466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298646754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[1]R flush_numbers[0..12]: [181, 184, 186, 186, 182, 183, 168, 179, 162, 183, 186, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298660882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[1]R generation[0..12]: [92, 94, 95, 95, 93, 94, 85, 90, 82, 94, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298679209Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298693677Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[2]R flush_numbers[0..12]: [181, 184, 186, 186, 182, 183, 168, 179, 162, 183, 186, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298707415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[2]R generation[0..12]: [92, 94, 95, 95, 93, 94, 85, 90, 82, 94, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298720943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298734891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298747549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298764247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298777525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298790433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"":"downstairs","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298803761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298815589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"0","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298828787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"1","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298841695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"client":"2","":"downstairs","session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298854873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298866901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"b74ea671-7523-4db5-ae9e-bfb305f291a8 is now active with session: 5b2addd1-11b4-4baf-b996-c54168ac4781","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298882349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298894157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567,"session_id":"5b2addd1-11b4-4baf-b996-c54168ac4781"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:01.298906855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 15 len 4096 data: 8 02/30 Flush 03/30 Flush 04/30 Write block 465 len 4096 data: 6 05/30 Write block 534 len 32768 data: 9 8 6 6 4 5 4 2 06/30 Read block 39 len 4096 07/30 Write block 441 len 40960 data: 7 9 9 7 7 7 6 8 8 8 08/30 Write block 256 len 24576 data: 8 10 12 10 7 8 09/30 Write block 483 len 32768 data: 10 12 13 11 11 14 14 11 10/30 Write block 426 len 36864 data: 8 9 8 8 9 10 10 10 10 11/30 Read block 295 len 8192 12/30 Read block 389 len 4096 13/30 Write block 385 len 8192 data: 6 7 14/30 Write block 265 len 24576 data: 8 5 6 6 8 6 15/30 Read block 329 len 28672 16/30 Read block 133 len 8192 17/30 Write block 158 len 20480 data: 8 8 11 10 13 18/30 Read block 85 len 28672 19/30 Write block 493 len 28672 data: 7 7 9 11 13 11 9 20/30 Write block 494 len 4096 data: 8 21/30 Flush 22/30 Write block 4 len 16384 data: 6 7 7 7 23/30 Write block 467 len 16384 data: 11 6 4 3 24/30 Write block 573 len 40960 data: 9 7 7 7 6 8 7 10 11 7 25/30 Flush 26/30 Write block 470 len 40960 data: 4 3 3 5 5 6 7 6 7 8 27/30 Read block 274 len 16384 28/30 Read block 47 len 12288 29/30 Read block 311 len 24576 30/30 Write block 334 len 4096 data: 9 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:01.326369075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1567} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 190 190 190 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 95 95 95 186 186 186 F F F 3 060-079 95 95 95 186 186 186 F F F 4 080-099 93 93 93 182 182 182 F F F 5 100-119 94 94 94 183 183 183 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 96 96 96 189 189 189 F F F 8 160-179 96 96 96 189 189 189 F F F 9 180-199 94 94 94 183 183 183 F F F 10 200-219 95 95 95 186 186 186 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 96 96 96 189 189 189 F F F 13 260-279 96 96 96 189 189 189 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 92 92 92 181 181 181 T T T 17 340-359 90 90 90 178 178 178 F F F 18 360-379 95 95 95 186 186 186 F F F 19 380-399 96 96 96 189 189 189 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 96 96 96 189 189 189 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 96 96 96 190 190 190 T T T 24 480-499 96 96 96 189 189 189 F F F 25 500-519 94 94 94 185 185 185 F F F 26 520-539 96 96 96 189 189 189 F F F 27 540-559 96 96 96 189 189 189 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 96 96 96 190 190 190 F F F Max gen: 96, Max flush: 190 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 190 190 190 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 95 95 95 186 186 186 F F F 3 060-079 95 95 95 186 186 186 F F F 4 080-099 93 93 93 182 182 182 F F F 5 100-119 94 94 94 183 183 183 F F F 6 120-139 85 85 85 168 168 168 F F F 7 140-159 96 96 96 189 189 189 F F F 8 160-179 96 96 96 189 189 189 F F F 9 180-199 94 94 94 183 183 183 F F F 10 200-219 95 95 95 186 186 186 F F F 11 220-239 92 92 92 181 181 181 F F F 12 240-259 96 96 96 189 189 189 F F F 13 260-279 96 96 96 189 189 189 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 97 97 97 191 191 191 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 95 95 95 186 186 186 F F F 19 380-399 96 96 96 189 189 189 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 96 96 96 189 189 189 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 97 97 97 191 191 191 F F F 24 480-499 96 96 96 189 189 189 F F F 25 500-519 94 94 94 185 185 185 F F F 26 520-539 96 96 96 189 189 189 F F F 27 540-559 96 96 96 189 189 189 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 96 96 96 190 190 190 F F F Max gen: 97, Max flush: 191 Begin loop 49 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.509104785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.509658023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.509691438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575} {"msg":"Crucible 4b4a71b8-a5bc-4b1f-bb2b-ee7a27128f88 has session id: baed7f02-ddd9-4759-839c-460ba484afba","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.509816759Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Upstairs opts: Upstairs UUID: 4b4a71b8-a5bc-4b1f-bb2b-ee7a27128f88, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.509858503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Crucible stats registered with UUID: 4b4a71b8-a5bc-4b1f-bb2b-ee7a27128f88","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.509882539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510172467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575} {"msg":"4b4a71b8-a5bc-4b1f-bb2b-ee7a27128f88 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510204812Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510229488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510255164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510280251Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510304667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510329713Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.51035389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510378856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510402822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510428709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:15.510514556Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:15.51055364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:15.510579486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510598044Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.51061952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_connection connected from Ok(127.0.0.1:48646)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510642607Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510665314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_connection connected from Ok(127.0.0.1:50027)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.510828969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.51149634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.511525646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.511541254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.511555072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512178649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512218143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512767062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512797467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512812855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512852249Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512868937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:15.512882135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.511766339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_connection connected from Ok(127.0.0.1:35166)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.511840339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"io task","client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.513010685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.513047789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.51365065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514253611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514276427Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[0]R flush_numbers[0..12]: [190, 184, 186, 186, 182, 183, 168, 189, 189, 183, 186, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514291425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[0]R generation[0..12]: [96, 94, 95, 95, 93, 94, 85, 96, 96, 94, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514305593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514319921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[1]R flush_numbers[0..12]: [190, 184, 186, 186, 182, 183, 168, 189, 189, 183, 186, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514333819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[1]R generation[0..12]: [96, 94, 95, 95, 93, 94, 85, 96, 96, 94, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514352346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514366784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[2]R flush_numbers[0..12]: [190, 184, 186, 186, 182, 183, 168, 189, 189, 183, 186, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514380592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[2]R generation[0..12]: [96, 94, 95, 95, 93, 94, 85, 96, 96, 94, 95, 92]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.51439417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514408078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514421066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514434144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:28:25.514446962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514474988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"":"downstairs","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514488756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514500894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"0","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514514512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"1","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.51452786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"client":"2","":"downstairs","session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514542958Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514557656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"4b4a71b8-a5bc-4b1f-bb2b-ee7a27128f88 is now active with session: baed7f02-ddd9-4759-839c-460ba484afba","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514575123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514591791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575,"session_id":"baed7f02-ddd9-4759-839c-460ba484afba"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:25.514611198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 275 len 8192 data: 11 11 02/30 Write block 550 len 12288 data: 5 4 3 03/30 Read block 290 len 40960 04/30 Read block 230 len 12288 05/30 Read block 155 len 40960 06/30 Read block 237 len 32768 07/30 Write block 273 len 20480 data: 8 10 12 12 12 08/30 Write block 548 len 12288 data: 5 5 6 09/30 Write block 363 len 40960 data: 6 8 9 11 13 12 11 10 10 8 10/30 Write block 122 len 16384 data: 5 6 4 5 11/30 Flush 12/30 Write block 221 len 28672 data: 9 6 5 3 4 4 4 13/30 Write block 423 len 24576 data: 7 7 8 9 10 9 14/30 Read block 586 len 32768 15/30 Flush 16/30 Write block 373 len 12288 data: 13 13 12 17/30 Flush 18/30 Write block 164 len 32768 data: 12 12 12 12 10 12 10 10 19/30 Write block 43 len 8192 data: 8 13 20/30 Read block 407 len 40960 21/30 Write block 422 len 8192 data: 5 8 22/30 Write block 205 len 32768 data: 9 11 13 11 10 10 12 10 23/30 Read block 101 len 16384 24/30 Write block 118 len 16384 data: 7 6 7 6 25/30 Write block 156 len 12288 data: 10 10 9 26/30 Write block 88 len 28672 data: 7 8 8 6 7 8 6 27/30 Read block 499 len 4096 28/30 Read block 95 len 28672 29/30 Read block 354 len 4096 30/30 Write block 241 len 16384 data: 5 6 5 6 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 8 Done Done Done false 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Write 2 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Read 7 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:25.539715287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1575} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 190 190 190 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 95 95 95 186 186 186 T T T 3 060-079 95 95 95 186 186 186 F F F 4 080-099 93 93 93 182 182 182 T T T 5 100-119 94 94 94 183 183 183 T T T 6 120-139 98 98 98 192 192 192 T T T 7 140-159 96 96 96 189 189 189 T T T 8 160-179 96 96 96 189 189 189 T T T 9 180-199 94 94 94 183 183 183 F F F 10 200-219 95 95 95 186 186 186 T T T 11 220-239 98 98 98 193 193 193 F F F 12 240-259 96 96 96 189 189 189 T T T 13 260-279 98 98 98 192 192 192 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 97 97 97 191 191 191 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 98 98 98 194 194 194 F F F 19 380-399 96 96 96 189 189 189 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 98 98 98 193 193 193 T T T 22 440-459 96 96 96 189 189 189 F F F 23 460-479 97 97 97 191 191 191 F F F 24 480-499 96 96 96 189 189 189 F F F 25 500-519 94 94 94 185 185 185 F F F 26 520-539 96 96 96 189 189 189 F F F 27 540-559 98 98 98 192 192 192 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 96 96 96 190 190 190 F F F Max gen: 98, Max flush: 194 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 190 190 190 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 99 99 99 195 195 195 F F F 3 060-079 95 95 95 186 186 186 F F F 4 080-099 99 99 99 195 195 195 F F F 5 100-119 99 99 99 195 195 195 F F F 6 120-139 99 99 99 195 195 195 F F F 7 140-159 99 99 99 195 195 195 F F F 8 160-179 99 99 99 195 195 195 F F F 9 180-199 94 94 94 183 183 183 F F F 10 200-219 99 99 99 195 195 195 F F F 11 220-239 98 98 98 193 193 193 F F F 12 240-259 99 99 99 195 195 195 F F F 13 260-279 98 98 98 192 192 192 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 92 92 92 181 181 181 F F F 16 320-339 97 97 97 191 191 191 F F F 17 340-359 90 90 90 178 178 178 F F F 18 360-379 98 98 98 194 194 194 F F F 19 380-399 96 96 96 189 189 189 F F F 20 400-419 95 95 95 186 186 186 F F F 21 420-439 99 99 99 195 195 195 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 97 97 97 191 191 191 F F F 24 480-499 96 96 96 189 189 189 F F F 25 500-519 94 94 94 185 185 185 F F F 26 520-539 96 96 96 189 189 189 F F F 27 540-559 98 98 98 192 192 192 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 96 96 96 190 190 190 F F F Max gen: 99, Max flush: 195 Begin loop 50 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.809688294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810151866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.81018838Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583} {"msg":"Crucible c59086dd-b264-44ce-8e7e-1891aec93abc has session id: b2275aa8-e185-4b9a-ab74-3ce481c2887b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810398699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Upstairs opts: Upstairs UUID: c59086dd-b264-44ce-8e7e-1891aec93abc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810435624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Crucible stats registered with UUID: c59086dd-b264-44ce-8e7e-1891aec93abc","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810451552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810847053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583} {"msg":"c59086dd-b264-44ce-8e7e-1891aec93abc active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810910093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.81093463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810950118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810963705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.810979903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.81100294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811022367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811036075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811049783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.81106671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:39.811438165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:39.81147525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:39.811493367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811505995Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811518853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811579114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62318)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811658833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49745)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.811679059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.812451575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.812478561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.812493159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.812506707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.81309473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.813122096Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.813621422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.813647088Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.813660736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.813673114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.813686782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:39.81369973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.812509716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36707)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.812615351Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"io task","client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.813642099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.813670094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814269146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814822264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814855799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[0]R flush_numbers[0..12]: [190, 184, 195, 186, 195, 195, 195, 195, 195, 183, 195, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814871916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[0]R generation[0..12]: [96, 94, 99, 95, 99, 99, 99, 99, 99, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814886654Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814905951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[1]R flush_numbers[0..12]: [190, 184, 195, 186, 195, 195, 195, 195, 195, 183, 195, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814920179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[1]R generation[0..12]: [96, 94, 99, 95, 99, 99, 99, 99, 99, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814934167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814948525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[2]R flush_numbers[0..12]: [190, 184, 195, 186, 195, 195, 195, 195, 195, 183, 195, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814962363Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[2]R generation[0..12]: [96, 94, 99, 95, 99, 99, 99, 99, 99, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814976081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.814990159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815003187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:28:49.815016695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Next flush: 196","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815032402Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815045571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"":"downstairs","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815058729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815070737Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"0","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815084075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"1","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815108591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"client":"2","":"downstairs","session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815134777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815157064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"c59086dd-b264-44ce-8e7e-1891aec93abc is now active with session: b2275aa8-e185-4b9a-ab74-3ce481c2887b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.81518147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815203817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583,"session_id":"b2275aa8-e185-4b9a-ab74-3ce481c2887b"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:28:49.815219155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 323 len 28672 data: 7 8 8 9 8 9 7 02/30 Write block 535 len 4096 data: 9 03/30 Read block 260 len 36864 04/30 Read block 563 len 4096 05/30 Write block 46 len 40960 data: 11 11 8 8 7 8 9 10 9 10 06/30 Flush 07/30 Write block 585 len 8192 data: 6 7 08/30 Read block 495 len 16384 09/30 Write block 411 len 8192 data: 10 9 10/30 Read block 485 len 28672 11/30 Read block 549 len 16384 12/30 Read block 230 len 36864 13/30 Flush 14/30 Write block 539 len 8192 data: 6 5 15/30 Write block 107 len 4096 data: 8 16/30 Read block 488 len 32768 17/30 Read block 307 len 20480 18/30 Read block 5 len 36864 19/30 Read block 431 len 16384 20/30 Flush 21/30 Write block 478 len 4096 data: 8 22/30 Write block 317 len 12288 data: 9 7 5 23/30 Flush 24/30 Write block 352 len 32768 data: 8 7 7 7 5 3 4 7 25/30 Write block 159 len 20480 data: 9 12 11 14 12 26/30 Read block 285 len 40960 27/30 Write block 244 len 8192 data: 7 6 28/30 Write block 501 len 12288 data: 9 9 11 29/30 Read block 348 len 12288 30/30 Read block 60 len 28672 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:28:49.840253384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1583} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 190 190 190 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 100 100 100 196 196 196 F F F 3 060-079 95 95 95 186 186 186 F F F 4 080-099 99 99 99 195 195 195 F F F 5 100-119 100 100 100 198 198 198 F F F 6 120-139 99 99 99 195 195 195 F F F 7 140-159 99 99 99 195 195 195 T T T 8 160-179 99 99 99 195 195 195 T T T 9 180-199 94 94 94 183 183 183 F F F 10 200-219 99 99 99 195 195 195 F F F 11 220-239 98 98 98 193 193 193 F F F 12 240-259 99 99 99 195 195 195 T T T 13 260-279 98 98 98 192 192 192 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 100 100 100 199 199 199 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 90 90 90 178 178 178 T T T 18 360-379 98 98 98 194 194 194 F F F 19 380-399 96 96 96 189 189 189 F F F 20 400-419 100 100 100 197 197 197 F F F 21 420-439 99 99 99 195 195 195 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 100 100 100 199 199 199 F F F 24 480-499 96 96 96 189 189 189 F F F 25 500-519 94 94 94 185 185 185 T T T 26 520-539 100 100 100 198 198 198 F F F 27 540-559 100 100 100 198 198 198 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 100 100 100 197 197 197 F F F Max gen: 100, Max flush: 199 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 96 96 96 190 190 190 F F F 1 020-039 94 94 94 184 184 184 F F F 2 040-059 100 100 100 196 196 196 F F F 3 060-079 95 95 95 186 186 186 F F F 4 080-099 99 99 99 195 195 195 F F F 5 100-119 100 100 100 198 198 198 F F F 6 120-139 99 99 99 195 195 195 F F F 7 140-159 101 101 101 200 200 200 F F F 8 160-179 101 101 101 200 200 200 F F F 9 180-199 94 94 94 183 183 183 F F F 10 200-219 99 99 99 195 195 195 F F F 11 220-239 98 98 98 193 193 193 F F F 12 240-259 101 101 101 200 200 200 F F F 13 260-279 98 98 98 192 192 192 F F F 14 280-299 93 93 93 182 182 182 F F F 15 300-319 100 100 100 199 199 199 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 101 101 101 200 200 200 F F F 18 360-379 98 98 98 194 194 194 F F F 19 380-399 96 96 96 189 189 189 F F F 20 400-419 100 100 100 197 197 197 F F F 21 420-439 99 99 99 195 195 195 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 100 100 100 199 199 199 F F F 24 480-499 96 96 96 189 189 189 F F F 25 500-519 101 101 101 200 200 200 F F F 26 520-539 100 100 100 198 198 198 F F F 27 540-559 100 100 100 198 198 198 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 100 100 100 197 197 197 F F F Max gen: 101, Max flush: 200 Begin loop 51 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.05613953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.056485409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.056519724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591} {"msg":"Crucible 8f0198e6-9b2d-445c-a6c8-117f4c1e4602 has session id: 8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.056855144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Upstairs opts: Upstairs UUID: 8f0198e6-9b2d-445c-a6c8-117f4c1e4602, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.056897218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Crucible stats registered with UUID: 8f0198e6-9b2d-445c-a6c8-117f4c1e4602","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.056912865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057310626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591} {"msg":"8f0198e6-9b2d-445c-a6c8-117f4c1e4602 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.05735405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057370908Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057395724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.05742245Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057447226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057471833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057532213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057587095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.05762275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057653116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:04.057681911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:04.057711847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:04.057740073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.05776094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057782826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42697)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057908988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42304)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057942643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.057969979Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.058599855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.058625782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.058640309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.058653977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059144754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059253748Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059592758Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059627013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059642671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059655339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059669117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:04.059682435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.059284713Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_connection connected from Ok(127.0.0.1:54386)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.059340204Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"io task","client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.060296763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.060324858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.060894164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061480407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061508263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[0]R flush_numbers[0..12]: [190, 184, 196, 186, 195, 198, 195, 200, 200, 183, 195, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061525171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[0]R generation[0..12]: [96, 94, 100, 95, 99, 100, 99, 101, 101, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061539928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061558086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[1]R flush_numbers[0..12]: [190, 184, 196, 186, 195, 198, 195, 200, 200, 183, 195, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061572193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[1]R generation[0..12]: [96, 94, 100, 95, 99, 100, 99, 101, 101, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061585802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.06159954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[2]R flush_numbers[0..12]: [190, 184, 196, 186, 195, 198, 195, 200, 200, 183, 195, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061613398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[2]R generation[0..12]: [96, 94, 100, 95, 99, 100, 99, 101, 101, 94, 99, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061626996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061640854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061657641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061670819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061683447Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061696235Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"":"downstairs","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061709323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061721182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"0","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.06173484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"1","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061751387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"client":"2","":"downstairs","session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061765095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061777083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"8f0198e6-9b2d-445c-a6c8-117f4c1e4602 is now active with session: 8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061789721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.06180149Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591,"session_id":"8cdeaaf7-4ce5-4a73-9fee-435fb406b5e5"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:14.061813578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 122 len 8192 data: 6 7 02/30 Read block 125 len 16384 03/30 Read block 367 len 28672 04/30 Write block 205 len 16384 data: 10 12 14 12 05/30 Write block 189 len 28672 data: 6 6 7 6 5 4 7 06/30 Write block 62 len 36864 data: 9 8 9 9 8 8 6 9 4 07/30 Read block 246 len 40960 08/30 Write block 361 len 40960 data: 9 7 7 9 10 12 14 13 12 11 09/30 Read block 32 len 8192 10/30 Read block 507 len 20480 11/30 Flush 12/30 Write block 18 len 28672 data: 4 5 5 6 7 8 8 13/30 Read block 75 len 36864 14/30 Read block 306 len 40960 15/30 Read block 224 len 32768 16/30 Write block 432 len 32768 data: 11 11 11 8 10 9 8 7 17/30 Read block 73 len 40960 18/30 Write block 291 len 24576 data: 12 10 7 8 8 8 19/30 Flush 20/30 Write block 509 len 28672 data: 11 10 10 10 7 10 10 21/30 Write block 38 len 32768 data: 2 2 3 5 8 9 14 11 22/30 Write block 380 len 8192 data: 13 11 23/30 Read block 431 len 20480 24/30 Read block 17 len 4096 25/30 Read block 376 len 8192 26/30 Write block 482 len 40960 data: 10 11 13 14 12 12 15 15 12 9 27/30 Write block 400 len 28672 data: 10 10 10 10 10 11 12 28/30 Write block 555 len 16384 data: 8 10 7 7 29/30 Write block 534 len 4096 data: 10 30/30 Write block 249 len 32768 data: 8 9 10 11 12 11 7 9 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 1 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Write 10 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Write 1 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:14.095616407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1591} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 202 202 202 F F F 1 020-039 102 102 102 202 202 202 T T T 2 040-059 100 100 100 196 196 196 T T T 3 060-079 102 102 102 201 201 201 F F F 4 080-099 99 99 99 195 195 195 F F F 5 100-119 100 100 100 198 198 198 F F F 6 120-139 102 102 102 201 201 201 F F F 7 140-159 101 101 101 200 200 200 F F F 8 160-179 101 101 101 200 200 200 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 102 102 102 201 201 201 F F F 11 220-239 98 98 98 193 193 193 F F F 12 240-259 101 101 101 200 200 200 T T T 13 260-279 98 98 98 192 192 192 F F F 14 280-299 102 102 102 203 203 203 F F F 15 300-319 100 100 100 199 199 199 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 101 101 101 200 200 200 F F F 18 360-379 102 102 102 201 201 201 F F F 19 380-399 96 96 96 189 189 189 T T T 20 400-419 100 100 100 197 197 197 T T T 21 420-439 102 102 102 203 203 203 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 100 100 100 199 199 199 F F F 24 480-499 96 96 96 189 189 189 T T T 25 500-519 101 101 101 200 200 200 T T T 26 520-539 100 100 100 198 198 198 T T T 27 540-559 100 100 100 198 198 198 T T T 28 560-579 96 96 96 190 190 190 F F F 29 580-599 100 100 100 197 197 197 F F F Max gen: 102, Max flush: 203 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 202 202 202 F F F 1 020-039 103 103 103 204 204 204 F F F 2 040-059 103 103 103 204 204 204 F F F 3 060-079 102 102 102 201 201 201 F F F 4 080-099 99 99 99 195 195 195 F F F 5 100-119 100 100 100 198 198 198 F F F 6 120-139 102 102 102 201 201 201 F F F 7 140-159 101 101 101 200 200 200 F F F 8 160-179 101 101 101 200 200 200 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 102 102 102 201 201 201 F F F 11 220-239 98 98 98 193 193 193 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 98 98 98 192 192 192 F F F 14 280-299 102 102 102 203 203 203 F F F 15 300-319 100 100 100 199 199 199 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 101 101 101 200 200 200 F F F 18 360-379 102 102 102 201 201 201 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 103 103 103 204 204 204 F F F 21 420-439 102 102 102 203 203 203 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 100 100 100 199 199 199 F F F 24 480-499 103 103 103 204 204 204 F F F 25 500-519 103 103 103 204 204 204 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 103 103 103 204 204 204 F F F 28 560-579 96 96 96 190 190 190 F F F 29 580-599 100 100 100 197 197 197 F F F Max gen: 103, Max flush: 204 Begin loop 52 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.370850087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.371662077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.371692612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599} {"msg":"Crucible 7f3d1bb4-b122-4ffc-a6ac-64f330f0c35f has session id: 7d18a05f-3d54-456d-b04e-6ea7acfe3a4c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.371716698Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Upstairs opts: Upstairs UUID: 7f3d1bb4-b122-4ffc-a6ac-64f330f0c35f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.371752953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Crucible stats registered with UUID: 7f3d1bb4-b122-4ffc-a6ac-64f330f0c35f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.371779279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372074505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599} {"msg":"7f3d1bb4-b122-4ffc-a6ac-64f330f0c35f active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372169122Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372187459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372202457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372216335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372229813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.37224298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372261768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372290253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372306621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372320429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55277)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372532977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:28.372697943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:28.372740707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:28.372774352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372800118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372826494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44970)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.37285366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.372879526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.373376883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.373403179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.373418086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.373440243Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374045053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374082048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374614389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374647564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.37467308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374687668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374701576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:28.374714784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.373768454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39533)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.373821066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"io task","client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.374954838Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.374982504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.375556289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376147901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376175577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[0]R flush_numbers[0..12]: [202, 204, 204, 201, 195, 198, 201, 200, 200, 201, 201, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376198484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[0]R generation[0..12]: [102, 103, 103, 102, 99, 100, 102, 101, 101, 102, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376217281Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376231919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[1]R flush_numbers[0..12]: [202, 204, 204, 201, 195, 198, 201, 200, 200, 201, 201, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376246007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[1]R generation[0..12]: [102, 103, 103, 102, 99, 100, 102, 101, 101, 102, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376259865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376274023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[2]R flush_numbers[0..12]: [202, 204, 204, 201, 195, 198, 201, 200, 200, 201, 201, 193]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376287541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[2]R generation[0..12]: [102, 103, 103, 102, 99, 100, 102, 101, 101, 102, 102, 98]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376301109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376314977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376332264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376345512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Next flush: 205","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.37635838Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376371479Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"":"downstairs","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376384506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376398415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"0","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376411613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"1","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376424511Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"client":"2","":"downstairs","session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376437589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376449767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"7f3d1bb4-b122-4ffc-a6ac-64f330f0c35f is now active with session: 7d18a05f-3d54-456d-b04e-6ea7acfe3a4c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376463215Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376474853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599,"session_id":"7d18a05f-3d54-456d-b04e-6ea7acfe3a4c"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:38.376486881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 156 len 20480 data: 11 11 10 10 13 02/30 Read block 198 len 24576 03/30 Read block 435 len 8192 04/30 Write block 55 len 16384 data: 11 9 9 8 05/30 Flush 06/30 Write block 75 len 40960 data: 4 6 7 9 8 7 6 7 8 7 07/30 Read block 546 len 28672 08/30 Read block 488 len 16384 09/30 Write block 288 len 8192 data: 10 10 10/30 Read block 178 len 20480 11/30 Write block 98 len 8192 data: 4 3 12/30 Flush 13/30 Write block 111 len 36864 data: 11 14 15 11 11 10 6 8 7 14/30 Write block 345 len 20480 data: 12 12 8 8 8 15/30 Write block 348 len 32768 data: 9 9 9 7 9 8 8 8 16/30 Write block 556 len 12288 data: 11 8 8 17/30 Read block 529 len 32768 18/30 Read block 68 len 16384 19/30 Write block 236 len 8192 data: 5 5 20/30 Write block 424 len 36864 data: 8 9 10 11 10 9 10 11 12 21/30 Read block 273 len 28672 22/30 Read block 196 len 12288 23/30 Write block 219 len 20480 data: 8 10 10 7 6 24/30 Write block 579 len 24576 data: 8 11 12 8 5 5 25/30 Write block 419 len 12288 data: 4 6 5 26/30 Flush 27/30 Write block 509 len 32768 data: 12 11 11 11 8 11 11 9 28/30 Write block 482 len 12288 data: 11 12 14 29/30 Read block 383 len 32768 30/30 Write block 299 len 24576 data: 9 8 7 7 7 8 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:38.404099718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1599} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 202 202 202 F F F 1 020-039 103 103 103 204 204 204 F F F 2 040-059 104 104 104 205 205 205 F F F 3 060-079 104 104 104 206 206 206 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 104 104 104 207 207 207 F F F 6 120-139 102 102 102 201 201 201 F F F 7 140-159 104 104 104 205 205 205 F F F 8 160-179 104 104 104 205 205 205 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 104 104 104 208 208 208 F F F 11 220-239 104 104 104 208 208 208 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 98 98 98 192 192 192 F F F 14 280-299 104 104 104 206 206 206 T T T 15 300-319 100 100 100 199 199 199 T T T 16 320-339 100 100 100 196 196 196 F F F 17 340-359 104 104 104 207 207 207 F F F 18 360-379 102 102 102 201 201 201 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 104 104 104 208 208 208 F F F 21 420-439 104 104 104 208 208 208 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 100 100 100 199 199 199 F F F 24 480-499 103 103 103 204 204 204 T T T 25 500-519 103 103 103 204 204 204 T T T 26 520-539 103 103 103 204 204 204 F F F 27 540-559 104 104 104 207 207 207 F F F 28 560-579 104 104 104 208 208 208 F F F 29 580-599 104 104 104 208 208 208 F F F Max gen: 104, Max flush: 208 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 202 202 202 F F F 1 020-039 103 103 103 204 204 204 F F F 2 040-059 104 104 104 205 205 205 F F F 3 060-079 104 104 104 206 206 206 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 104 104 104 207 207 207 F F F 6 120-139 102 102 102 201 201 201 F F F 7 140-159 104 104 104 205 205 205 F F F 8 160-179 104 104 104 205 205 205 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 104 104 104 208 208 208 F F F 11 220-239 104 104 104 208 208 208 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 98 98 98 192 192 192 F F F 14 280-299 105 105 105 209 209 209 F F F 15 300-319 105 105 105 209 209 209 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 104 104 104 207 207 207 F F F 18 360-379 102 102 102 201 201 201 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 104 104 104 208 208 208 F F F 21 420-439 104 104 104 208 208 208 F F F 22 440-459 96 96 96 189 189 189 F F F 23 460-479 100 100 100 199 199 199 F F F 24 480-499 105 105 105 209 209 209 F F F 25 500-519 105 105 105 209 209 209 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 104 104 104 207 207 207 F F F 28 560-579 104 104 104 208 208 208 F F F 29 580-599 104 104 104 208 208 208 F F F Max gen: 105, Max flush: 209 Begin loop 53 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.611596757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612100372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612127108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607} {"msg":"Crucible 6af03317-ba9e-46ca-8a5d-131b67d2b14b has session id: 316083cd-50cd-458d-8eb3-68e84cac246c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612282245Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Upstairs opts: Upstairs UUID: 6af03317-ba9e-46ca-8a5d-131b67d2b14b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.61238856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Crucible stats registered with UUID: 6af03317-ba9e-46ca-8a5d-131b67d2b14b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612418535Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612781641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607} {"msg":"6af03317-ba9e-46ca-8a5d-131b67d2b14b active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612869778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612887926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612902894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612916591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612929909Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612952176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.612974403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613027385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613064319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613092975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53164)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613386282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34437)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613492796Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:52.613615298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:52.613646793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:29:52.613677679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613698885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613724062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613832066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.613995341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.614028277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.614044714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.614058452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.614643285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.614780325Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.615171257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.615206272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.615229598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.615254985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.615277901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:29:52.615299188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.614785935Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48758)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.614935833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"io task","client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.615989236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.616022132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.616631231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617184949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617205066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[0]R flush_numbers[0..12]: [202, 204, 205, 206, 206, 207, 201, 205, 205, 201, 208, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617225013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[0]R generation[0..12]: [102, 103, 104, 104, 104, 104, 102, 104, 104, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617239641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617254999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[1]R flush_numbers[0..12]: [202, 204, 205, 206, 206, 207, 201, 205, 205, 201, 208, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617269607Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[1]R generation[0..12]: [102, 103, 104, 104, 104, 104, 102, 104, 104, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617283765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617297922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[2]R flush_numbers[0..12]: [202, 204, 205, 206, 206, 207, 201, 205, 205, 201, 208, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.61731186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[2]R generation[0..12]: [102, 103, 104, 104, 104, 104, 102, 104, 104, 102, 104, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617325908Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617339876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617352804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Generation requested: 106 >= found:106","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617369372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Next flush: 210","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.61738248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617395328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"":"downstairs","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617408376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617422584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"0","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617435802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"1","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.6174487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"client":"2","":"downstairs","session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617461878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617473896Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"6af03317-ba9e-46ca-8a5d-131b67d2b14b is now active with session: 316083cd-50cd-458d-8eb3-68e84cac246c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617486684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617498403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607,"session_id":"316083cd-50cd-458d-8eb3-68e84cac246c"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:02.617510361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 481 len 24576 data: 9 12 13 15 15 13 02/30 Read block 235 len 32768 03/30 Read block 579 len 16384 04/30 Flush 05/30 Write block 205 len 12288 data: 11 13 15 06/30 Write block 437 len 16384 data: 10 9 8 6 07/30 Flush 08/30 Write block 273 len 4096 data: 9 09/30 Read block 586 len 40960 10/30 Read block 121 len 12288 11/30 Write block 354 len 40960 data: 9 9 6 4 5 8 9 10 8 8 12/30 Read block 113 len 36864 13/30 Read block 94 len 28672 14/30 Write block 421 len 32768 data: 6 6 9 9 10 11 12 11 15/30 Read block 178 len 28672 16/30 Write block 466 len 12288 data: 8 12 7 17/30 Read block 432 len 32768 18/30 Write block 313 len 16384 data: 8 8 10 10 19/30 Write block 401 len 4096 data: 11 20/30 Read block 542 len 24576 21/30 Write block 163 len 32768 data: 13 13 13 13 13 11 13 11 22/30 Read block 272 len 16384 23/30 Write block 18 len 24576 data: 5 6 6 7 8 9 24/30 Write block 540 len 16384 data: 6 3 4 4 25/30 Read block 251 len 40960 26/30 Read block 282 len 40960 27/30 Read block 121 len 4096 28/30 Read block 406 len 32768 29/30 Write block 60 len 40960 data: 10 10 10 9 10 10 9 9 7 10 30/30 Read block 280 len 20480 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 10 Done Done Done false 31 Acked 1030 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 12 11 11 34 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:02.645068146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1607} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 202 202 202 T T T 1 020-039 103 103 103 204 204 204 T T T 2 040-059 104 104 104 205 205 205 F F F 3 060-079 104 104 104 206 206 206 T T T 4 080-099 104 104 104 206 206 206 F F F 5 100-119 104 104 104 207 207 207 F F F 6 120-139 102 102 102 201 201 201 F F F 7 140-159 104 104 104 205 205 205 F F F 8 160-179 104 104 104 205 205 205 T T T 9 180-199 102 102 102 201 201 201 F F F 10 200-219 106 106 106 211 211 211 F F F 11 220-239 104 104 104 208 208 208 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 105 105 105 209 209 209 F F F 15 300-319 106 106 106 212 212 212 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 106 106 106 212 212 212 F F F 18 360-379 106 106 106 212 212 212 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 104 104 104 208 208 208 T T T 21 420-439 106 106 106 212 212 212 F F F 22 440-459 106 106 106 211 211 211 F F F 23 460-479 106 106 106 212 212 212 F F F 24 480-499 106 106 106 210 210 210 F F F 25 500-519 105 105 105 209 209 209 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 104 104 104 207 207 207 T T T 28 560-579 104 104 104 208 208 208 F F F 29 580-599 104 104 104 208 208 208 F F F Max gen: 106, Max flush: 212 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 104 104 104 205 205 205 F F F 3 060-079 107 107 107 213 213 213 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 104 104 104 207 207 207 F F F 6 120-139 102 102 102 201 201 201 F F F 7 140-159 104 104 104 205 205 205 F F F 8 160-179 107 107 107 213 213 213 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 106 106 106 211 211 211 F F F 11 220-239 104 104 104 208 208 208 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 105 105 105 209 209 209 F F F 15 300-319 106 106 106 212 212 212 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 106 106 106 212 212 212 F F F 18 360-379 106 106 106 212 212 212 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 107 107 107 213 213 213 F F F 21 420-439 106 106 106 212 212 212 F F F 22 440-459 106 106 106 211 211 211 F F F 23 460-479 106 106 106 212 212 212 F F F 24 480-499 106 106 106 210 210 210 F F F 25 500-519 105 105 105 209 209 209 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 107 107 107 213 213 213 F F F 28 560-579 104 104 104 208 208 208 F F F 29 580-599 104 104 104 208 208 208 F F F Max gen: 107, Max flush: 213 Begin loop 54 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.874561484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875035334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.87506458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615} {"msg":"Crucible 07aa6279-9647-4d0e-b969-9f84d7725f9a has session id: a087bc26-164c-4c1f-ab5a-265ab8476eac","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875211018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Upstairs opts: Upstairs UUID: 07aa6279-9647-4d0e-b969-9f84d7725f9a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875259151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Crucible stats registered with UUID: 07aa6279-9647-4d0e-b969-9f84d7725f9a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875286517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875568415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615} {"msg":"07aa6279-9647-4d0e-b969-9f84d7725f9a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.87567085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875699016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875722272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875744259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875766326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875792032Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875814319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875835225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875857472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.875879539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:16.875985153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:61479)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876035856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:16.876062082Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:16.876087388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876106925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876181634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:58870)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876214309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876437686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876830998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876869682Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876905497Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.876934593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.877480382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.877519096Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.878071954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.8780976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.878111968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.878124506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.878137774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:16.878151182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.876785714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:49992)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.876842965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"io task","client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.877968928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.87802383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.878581837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879126927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879162851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[0]R flush_numbers[0..12]: [213, 213, 205, 213, 206, 207, 201, 205, 213, 201, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879179369Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[0]R generation[0..12]: [107, 107, 104, 107, 104, 104, 102, 104, 107, 102, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879193997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879208095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[1]R flush_numbers[0..12]: [213, 213, 205, 213, 206, 207, 201, 205, 213, 201, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879221762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[1]R generation[0..12]: [107, 107, 104, 107, 104, 104, 102, 104, 107, 102, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879235291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879249098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[2]R flush_numbers[0..12]: [213, 213, 205, 213, 206, 207, 201, 205, 213, 201, 211, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879262667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[2]R generation[0..12]: [107, 107, 104, 107, 104, 104, 102, 104, 107, 102, 106, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879276185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879289992Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"Wait for a query_work_queue command to finish before sending IO } {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.87930736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879320958Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Next flush: 214","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879333926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879351173Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"":"downstairs","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879364801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.87937655Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"0","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879389708Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"1","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879402886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"client":"2","":"downstairs","session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879416374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879428462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"07aa6279-9647-4d0e-b969-9f84d7725f9a is now active with session: a087bc26-164c-4c1f-ab5a-265ab8476eac","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.87944119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879453008Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615,"session_id":"a087bc26-164c-4c1f-ab5a-265ab8476eac"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:26.879465086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 297 len 28672 data: 9 8 10 9 8 8 8 02/30 Write block 202 len 16384 data: 8 7 8 12 03/30 Write block 438 len 32768 data: 10 9 7 8 10 10 8 8 04/30 Write block 60 len 24576 data: 11 11 11 10 11 11 05/30 Read block 537 len 36864 06/30 Read block 116 len 16384 07/30 Read block 439 len 20480 08/30 Write block 568 len 40960 data: 10 8 8 11 10 10 8 8 8 7 09/30 Write block 351 len 20480 data: 8 10 9 10 10 10/30 Read block 197 len 8192 11/30 Read block 587 len 32768 12/30 Read block 109 len 20480 13/30 Read block 102 len 4096 14/30 Write block 543 len 20480 data: 5 4 4 5 5 15/30 Read block 471 len 4096 16/30 Read block 160 len 8192 17/30 Flush 18/30 Write block 585 len 16384 data: 7 8 6 6 19/30 Read block 154 len 40960 20/30 Write block 64 len 4096 data: 12 21/30 Read block 436 len 24576 22/30 Read block 327 len 16384 23/30 Read block 197 len 40960 24/30 Read block 61 len 24576 25/30 Read block 456 len 8192 26/30 Read block 66 len 36864 27/30 Read block 389 len 4096 28/30 Write block 119 len 16384 data: 8 8 7 7 29/30 Read block 389 len 24576 30/30 Read block 514 len 12288 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 4 Done Done Done false 19 Acked 1018 Read 10 Done Done Done false 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Read 6 Done Done Done false 22 Acked 1021 Read 4 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 2 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 1 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 0 1 Done 12 13 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:26.904365726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1615} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 104 104 104 205 205 205 F F F 3 060-079 108 108 108 214 214 214 T T T 4 080-099 104 104 104 206 206 206 F F F 5 100-119 104 104 104 207 207 207 T T T 6 120-139 102 102 102 201 201 201 T T T 7 140-159 104 104 104 205 205 205 F F F 8 160-179 107 107 107 213 213 213 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 108 108 108 214 214 214 F F F 11 220-239 104 104 104 208 208 208 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 108 108 108 214 214 214 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 108 108 108 214 214 214 F F F 18 360-379 106 106 106 212 212 212 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 107 107 107 213 213 213 F F F 21 420-439 108 108 108 214 214 214 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 106 106 106 212 212 212 F F F 24 480-499 106 106 106 210 210 210 F F F 25 500-519 105 105 105 209 209 209 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 108 108 108 214 214 214 F F F 28 560-579 108 108 108 214 214 214 F F F 29 580-599 104 104 104 208 208 208 T T T Max gen: 108, Max flush: 214 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 104 104 104 205 205 205 F F F 3 060-079 109 109 109 215 215 215 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 109 109 109 215 215 215 F F F 6 120-139 109 109 109 215 215 215 F F F 7 140-159 104 104 104 205 205 205 F F F 8 160-179 107 107 107 213 213 213 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 108 108 108 214 214 214 F F F 11 220-239 104 104 104 208 208 208 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 108 108 108 214 214 214 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 108 108 108 214 214 214 F F F 18 360-379 106 106 106 212 212 212 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 107 107 107 213 213 213 F F F 21 420-439 108 108 108 214 214 214 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 106 106 106 212 212 212 F F F 24 480-499 106 106 106 210 210 210 F F F 25 500-519 105 105 105 209 209 209 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 108 108 108 214 214 214 F F F 28 560-579 108 108 108 214 214 214 F F F 29 580-599 109 109 109 215 215 215 F F F Max gen: 109, Max flush: 215 Begin loop 55 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.107623183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108078546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108104262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623} {"msg":"Crucible 5179a0d5-8bed-4ac1-8c09-66a06caf7021 has session id: 03092ff6-65d0-4fd3-a93b-ca5b5a21a416","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108279526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Upstairs opts: Upstairs UUID: 5179a0d5-8bed-4ac1-8c09-66a06caf7021, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108337438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Crucible stats registered with UUID: 5179a0d5-8bed-4ac1-8c09-66a06caf7021","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108360984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108784881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623} {"msg":"5179a0d5-8bed-4ac1-8c09-66a06caf7021 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108817997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108840853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108865359Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108888736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108918302Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108943578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108970234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.108999339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.109026646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.109053611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_connection connected from Ok(127.0.0.1:43180)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.109079508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_connection connected from Ok(127.0.0.1:60899)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.109109533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:41.109233325Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:41.109315263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:41.109456782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.109487777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.109510554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.10953703Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.110022188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.110061082Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.110091798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.110119753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.110568777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.110677881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.111097409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.111121725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.111135723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.111160229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.111174297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:41.111187515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.110325873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_connection connected from Ok(127.0.0.1:44014)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.110453194Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"io task","client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.111498999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.111526755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112185747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.11277338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112801406Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[0]R flush_numbers[0..12]: [213, 213, 205, 215, 206, 215, 215, 205, 213, 201, 214, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112818293Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[0]R generation[0..12]: [107, 107, 104, 109, 104, 109, 109, 104, 107, 102, 108, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112833361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112847949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[1]R flush_numbers[0..12]: [213, 213, 205, 215, 206, 215, 215, 205, 213, 201, 214, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112862147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[1]R generation[0..12]: [107, 107, 104, 109, 104, 109, 109, 104, 107, 102, 108, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112876005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112889853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[2]R flush_numbers[0..12]: [213, 213, 205, 215, 206, 215, 215, 205, 213, 201, 214, 208]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112903431Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[2]R generation[0..12]: [107, 107, 104, 109, 104, 109, 109, 104, 107, 102, 108, 104]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112917059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112930897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","Wait for a query_work_queue command to finish before sending IO pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112946974Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112965282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.112988378Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113013365Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"":"downstairs","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113035731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113057468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"0","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113080915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"1","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113101142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"client":"2","":"downstairs","session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113118979Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113131177Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"5179a0d5-8bed-4ac1-8c09-66a06caf7021 is now active with session: 03092ff6-65d0-4fd3-a93b-ca5b5a21a416","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113143965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113155603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623,"session_id":"03092ff6-65d0-4fd3-a93b-ca5b5a21a416"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:30:51.113167632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 8192 data: 9 8 02/30 Write block 300 len 8192 data: 10 9 03/30 Read block 598 len 4096 04/30 Read block 38 len 4096 05/30 Write block 563 len 4096 data: 11 06/30 Read block 247 len 4096 07/30 Read block 46 len 24576 08/30 Read block 544 len 40960 09/30 Flush 10/30 Write block 409 len 32768 data: 9 9 11 10 7 6 6 8 11/30 Read block 155 len 16384 12/30 Read block 330 len 12288 13/30 Flush 14/30 Write block 55 len 24576 data: 12 10 10 9 8 12 15/30 Read block 523 len 36864 16/30 Read block 383 len 28672 17/30 Flush 18/30 Write block 545 len 4096 data: 5 19/30 Read block 363 len 32768 20/30 Read block 76 len 32768 21/30 Write block 41 len 12288 data: 6 9 10 22/30 Write block 302 len 24576 data: 9 9 9 8 6 6 23/30 Flush 24/30 Write block 232 len 12288 data: 7 6 6 25/30 Read block 269 len 28672 26/30 Write block 494 len 32768 data: 9 10 12 14 12 10 10 10 27/30 Write block 113 len 24576 data: 16 12 12 11 7 9 28/30 Write block 157 len 36864 data: 12 11 11 14 12 15 14 14 14 29/30 Write block 566 len 28672 data: 8 8 11 9 9 12 11 30/30 Read block 293 len 24576 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 3 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Read 6 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:30:51.139043006Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1623} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 110 110 110 219 219 219 F F F 3 060-079 110 110 110 218 218 218 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 109 109 109 215 215 215 T T T 6 120-139 109 109 109 215 215 215 F F F 7 140-159 104 104 104 205 205 205 T T T 8 160-179 107 107 107 213 213 213 T T T 9 180-199 102 102 102 201 201 201 F F F 10 200-219 108 108 108 214 214 214 F F F 11 220-239 104 104 104 208 208 208 T T T 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 110 110 110 220 220 220 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 108 108 108 214 214 214 F F F 18 360-379 106 106 106 212 212 212 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 110 110 110 217 217 217 F F F 21 420-439 108 108 108 214 214 214 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 106 106 106 212 212 212 F F F 24 480-499 106 106 106 210 210 210 T T T 25 500-519 105 105 105 209 209 209 T T T 26 520-539 103 103 103 204 204 204 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 110 110 110 216 216 216 T T T 29 580-599 110 110 110 216 216 216 F F F Max gen: 110, Max flush: 220 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 110 110 110 219 219 219 F F F 3 060-079 110 110 110 218 218 218 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 111 111 111 221 221 221 F F F 6 120-139 109 109 109 215 215 215 F F F 7 140-159 111 111 111 221 221 221 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 108 108 108 214 214 214 F F F 11 220-239 111 111 111 221 221 221 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 110 110 110 220 220 220 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 108 108 108 214 214 214 F F F 18 360-379 106 106 106 212 212 212 F F F 19 380-399 103 103 103 204 204 204 F F F 20 400-419 110 110 110 217 217 217 F F F 21 420-439 108 108 108 214 214 214 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 106 106 106 212 212 212 F F F 24 480-499 111 111 111 221 221 221 F F F 25 500-519 111 111 111 221 221 221 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 111 111 111 221 221 221 F F F 29 580-599 110 110 110 216 216 216 F F F Max gen: 111, Max flush: 221 Begin loop 56 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.381831584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382540299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382566825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631} {"msg":"Crucible 31dbbbb0-cc5f-431a-84fd-864f055c44ea has session id: 80ca7b60-4f63-4a9f-9708-9c0d3815b4f8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382587932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Upstairs opts: Upstairs UUID: 31dbbbb0-cc5f-431a-84fd-864f055c44ea, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382619067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Crucible stats registered with UUID: 31dbbbb0-cc5f-431a-84fd-864f055c44ea","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382642274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38280151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631} {"msg":"31dbbbb0-cc5f-431a-84fd-864f055c44ea active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38286869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382904415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382931821Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382962726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.382987643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38300969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383033056Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383055613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383078059Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383100916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:05.38320533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:05.383244575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:57010)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38327228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:05.383300016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383331822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383358778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54751)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.383385574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38354696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38408518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.384118855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.384141762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.384166288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.384746402Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.384774158Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.385256286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.385282773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.385297091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.385336414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.385353232Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:05.38536656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.383922695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_connection connected from Ok(127.0.0.1:46056)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.383977186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"io task","client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.385182078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.38523794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.385813784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386398228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386424513Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[0]R flush_numbers[0..12]: [213, 213, 219, 218, 206, 221, 215, 221, 221, 201, 214, 221]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386440371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[0]R generation[0..12]: [107, 107, 110, 110, 104, 111, 109, 111, 111, 102, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386454919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386469097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[1]R flush_numbers[0..12]: [213, 213, 219, 218, 206, 221, 215, 221, 221, 201, 214, 221]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386483165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[1]R generation[0..12]: [107, 107, 110, 110, 104, 111, 109, 111, 111, 102, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386496883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386510881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[2]R flush_numbers[0..12]: [213, 213, 219, 218, 206, 221, 215, 221, 221, 201, 214, 221]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386524479Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[2]R generation[0..12]: [107, 107, 110, 110, 104, 111, 109, 111, 111, 102, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386538247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386552225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386565173Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.38658208Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Next flush: 222","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-08-05T19:31:15.386599408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386612636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"":"downstairs","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386625984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386637902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"0","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.38665103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"1","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386664108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"client":"2","":"downstairs","session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386677506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386689644Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"31dbbbb0-cc5f-431a-84fd-864f055c44ea is now active with session: 80ca7b60-4f63-4a9f-9708-9c0d3815b4f8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386702332Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386714071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631,"session_id":"80ca7b60-4f63-4a9f-9708-9c0d3815b4f8"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:15.386726169Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 16384 data: 10 11 12 14 02/30 Read block 242 len 12288 03/30 Read block 429 len 40960 04/30 Write block 404 len 32768 data: 11 12 13 10 8 10 10 12 05/30 Read block 451 len 20480 06/30 Read block 143 len 24576 07/30 Write block 476 len 28672 data: 8 7 9 9 10 10 13 08/30 Write block 595 len 12288 data: 9 7 6 09/30 Read block 429 len 20480 10/30 Read block 142 len 16384 11/30 Write block 482 len 12288 data: 14 14 16 12/30 Read block 297 len 32768 13/30 Read block 82 len 28672 14/30 Read block 170 len 8192 15/30 Write block 468 len 36864 data: 8 5 5 4 4 6 6 7 9 16/30 Read block 173 len 28672 17/30 Read block 305 len 40960 18/30 Read block 113 len 20480 19/30 Read block 242 len 36864 20/30 Read block 197 len 12288 21/30 Read block 98 len 20480 22/30 Flush 23/30 Write block 429 len 4096 data: 10 24/30 Read block 556 len 32768 25/30 Write block 124 len 16384 data: 5 6 6 8 26/30 Read block 576 len 32768 27/30 Write block 301 len 20480 data: 10 10 10 10 9 28/30 Write block 350 len 8192 data: 10 9 29/30 Write block 413 len 4096 data: 8 30/30 Read block 536 len 8192 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 1 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 8 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Write 1 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:15.410106393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1631} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 110 110 110 219 219 219 F F F 3 060-079 110 110 110 218 218 218 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 111 111 111 221 221 221 F F F 6 120-139 109 109 109 215 215 215 T T T 7 140-159 111 111 111 221 221 221 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 108 108 108 214 214 214 F F F 11 220-239 111 111 111 221 221 221 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 110 110 110 220 220 220 T T T 16 320-339 100 100 100 196 196 196 F F F 17 340-359 108 108 108 214 214 214 T T T 18 360-379 112 112 112 222 222 222 F F F 19 380-399 112 112 112 222 222 222 F F F 20 400-419 112 112 112 222 222 222 T T T 21 420-439 108 108 108 214 214 214 T T T 22 440-459 108 108 108 214 214 214 F F F 23 460-479 112 112 112 222 222 222 F F F 24 480-499 112 112 112 222 222 222 F F F 25 500-519 111 111 111 221 221 221 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 111 111 111 221 221 221 F F F 29 580-599 112 112 112 222 222 222 F F F Max gen: 112, Max flush: 222 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 107 107 107 213 213 213 F F F 2 040-059 110 110 110 219 219 219 F F F 3 060-079 110 110 110 218 218 218 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 111 111 111 221 221 221 F F F 6 120-139 113 113 113 223 223 223 F F F 7 140-159 111 111 111 221 221 221 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 108 108 108 214 214 214 F F F 11 220-239 111 111 111 221 221 221 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 113 113 113 223 223 223 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 113 113 113 223 223 223 F F F 18 360-379 112 112 112 222 222 222 F F F 19 380-399 112 112 112 222 222 222 F F F 20 400-419 113 113 113 223 223 223 F F F 21 420-439 113 113 113 223 223 223 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 112 112 112 222 222 222 F F F 24 480-499 112 112 112 222 222 222 F F F 25 500-519 111 111 111 221 221 221 F F F 26 520-539 103 103 103 204 204 204 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 111 111 111 221 221 221 F F F 29 580-599 112 112 112 222 222 222 F F F Max gen: 113, Max flush: 223 Begin loop 57 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.622692309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623285151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623313736Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639} {"msg":"Crucible 35e825c1-cb54-4225-824e-41976d80ed36 has session id: 61205dbd-47ad-4db0-bcb1-5a8c746b9e45","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623335153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Upstairs opts: Upstairs UUID: 35e825c1-cb54-4225-824e-41976d80ed36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623369908Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Crucible stats registered with UUID: 35e825c1-cb54-4225-824e-41976d80ed36","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623393355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623774778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639} {"msg":"35e825c1-cb54-4225-824e-41976d80ed36 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623861055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623880392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.62389538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623922276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623949732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.623976798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624000965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624044128Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624069455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624093841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_connection connected from Ok(127.0.0.1:48305)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624124966Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:29.624359532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_connection connected from Ok(127.0.0.1:46013)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624392267Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:29.624409704Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:29.624473664Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624491772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.62450526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.624578969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.625010605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.625037771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.625052009Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.625065337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.625609226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.62572007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.626144827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.626174852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.62618975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.626203948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.626217526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:29.626230514Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.62564168Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_connection connected from Ok(127.0.0.1:51745)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.62571429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"io task","client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.626825065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.62685843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.627528011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628126552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628154318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[0]R flush_numbers[0..12]: [213, 213, 219, 218, 206, 221, 223, 221, 221, 201, 214, 221]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628170735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[0]R generation[0..12]: [107, 107, 110, 110, 104, 111, 113, 111, 111, 102, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628185293Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628199551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[1]R flush_numbers[0..12]: [213, 213, 219, 218, 206, 221, 223, 221, 221, 201, 214, 221]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628213349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[1]R generation[0..12]: [107, 107, 110, 110, 104, 111, 113, 111, 111, 102, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628227117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628241005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[2]R flush_numbers[0..12]: [213, 213, 219, 218, 206, 221, 223, 221, 221, 201, 214, 221]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628254643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[2]R generation[0..12]: [107, 107, 110, 110, 104, 111, 113, 111, 111, 102, 108, 111]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628268771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628282879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628302366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628315814Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Next flush: 224","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628328592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.62834143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"":"downstairs","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628354398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628366226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"0","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628379124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"1","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628392002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"client":"2","":"downstairs","session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628405001Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628417189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"35e825c1-cb54-4225-824e-41976d80ed36 is now active with session: 61205dbd-47ad-4db0-bcb1-5a8c746b9e45","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628429897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628441685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639,"session_id":"61205dbd-47ad-4db0-bcb1-5a8c746b9e45"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:39.628453733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 8192 data: 13 12 02/30 Flush 03/30 Write block 473 len 32768 data: 7 7 8 10 8 10 10 11 04/30 Write block 200 len 36864 data: 8 7 9 8 9 13 14 16 13 05/30 Write block 306 len 32768 data: 7 7 5 6 9 10 9 9 06/30 Read block 286 len 32768 07/30 Read block 590 len 32768 08/30 Read block 331 len 20480 09/30 Write block 574 len 20480 data: 9 9 9 8 9 10/30 Write block 43 len 20480 data: 11 15 12 12 12 11/30 Read block 273 len 28672 12/30 Write block 496 len 24576 data: 13 15 13 11 11 11 13/30 Read block 253 len 36864 14/30 Read block 538 len 24576 15/30 Read block 375 len 16384 16/30 Write block 216 len 20480 data: 8 8 7 9 11 17/30 Write block 23 len 8192 data: 10 9 18/30 Write block 531 len 4096 data: 7 19/30 Read block 397 len 16384 20/30 Write block 214 len 24576 data: 8 9 9 9 8 10 21/30 Write block 494 len 40960 data: 10 11 14 16 14 12 12 12 10 12 22/30 Read block 39 len 24576 23/30 Read block 556 len 8192 24/30 Write block 520 len 36864 data: 8 8 7 8 8 6 5 6 5 25/30 Read block 129 len 28672 26/30 Write block 145 len 4096 data: 11 27/30 Read block 423 len 8192 28/30 Write block 396 len 28672 data: 12 9 9 9 11 12 11 29/30 Read block 81 len 40960 30/30 Write block 574 len 32768 data: 10 10 10 9 10 9 12 13 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Read 6 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 1 Done Done Done false 28 Acked 1027 Read 2 Done Done Done false 29 Acked 1028 Write 7 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:39.655389531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1639} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 114 114 114 225 225 225 F F F 2 040-059 114 114 114 225 225 225 F F F 3 060-079 110 110 110 218 218 218 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 111 111 111 221 221 221 F F F 6 120-139 113 113 113 223 223 223 F F F 7 140-159 111 111 111 221 221 221 T T T 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 114 114 114 225 225 225 F F F 11 220-239 114 114 114 225 225 225 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 114 114 114 225 225 225 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 113 113 113 223 223 223 F F F 18 360-379 112 112 112 222 222 222 F F F 19 380-399 112 112 112 222 222 222 T T T 20 400-419 113 113 113 223 223 223 T T T 21 420-439 113 113 113 223 223 223 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 114 114 114 225 225 225 F F F 24 480-499 114 114 114 225 225 225 T T T 25 500-519 114 114 114 225 225 225 T T T 26 520-539 114 114 114 225 225 225 T T T 27 540-559 110 110 110 219 219 219 F F F 28 560-579 114 114 114 225 225 225 T T T 29 580-599 112 112 112 222 222 222 T T T Max gen: 114, Max flush: 225 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 114 114 114 225 225 225 F F F 2 040-059 114 114 114 225 225 225 F F F 3 060-079 110 110 110 218 218 218 F F F 4 080-099 104 104 104 206 206 206 F F F 5 100-119 111 111 111 221 221 221 F F F 6 120-139 113 113 113 223 223 223 F F F 7 140-159 115 115 115 226 226 226 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 114 114 114 225 225 225 F F F 11 220-239 114 114 114 225 225 225 F F F 12 240-259 103 103 103 204 204 204 F F F 13 260-279 106 106 106 212 212 212 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 114 114 114 225 225 225 F F F 16 320-339 100 100 100 196 196 196 F F F 17 340-359 113 113 113 223 223 223 F F F 18 360-379 112 112 112 222 222 222 F F F 19 380-399 115 115 115 226 226 226 F F F 20 400-419 115 115 115 226 226 226 F F F 21 420-439 113 113 113 223 223 223 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 114 114 114 225 225 225 F F F 24 480-499 115 115 115 226 226 226 F F F 25 500-519 115 115 115 226 226 226 F F F 26 520-539 115 115 115 226 226 226 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 115 115 115 226 226 226 F F F 29 580-599 115 115 115 226 226 226 F F F Max gen: 115, Max flush: 226 Begin loop 58 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.911093534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.911806369Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.911837114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647} {"msg":"Crucible 6ddf3df0-5952-4c17-8e41-1ee012d1a25a has session id: 1c5d5186-2334-44fc-8d78-3bfeab1d245b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.911859271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Upstairs opts: Upstairs UUID: 6ddf3df0-5952-4c17-8e41-1ee012d1a25a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.911889856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Crucible stats registered with UUID: 6ddf3df0-5952-4c17-8e41-1ee012d1a25a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.911912843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912192521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647} {"msg":"6ddf3df0-5952-4c17-8e41-1ee012d1a25a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912300915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912331221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912360377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912385133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912409419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912434426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912458842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912482778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912508384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912532421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42106)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912620518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58880)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912797642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:53.912839715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:53.912871571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:31:53.912902696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912925023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.91294306Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.912991663Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.913512196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.913542421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.913567698Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.913594694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.914147522Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.914185596Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.91470068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.914725596Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.914739924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.914800265Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.914818472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:31:53.91483197Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.91375318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47991)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.913877952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"io task","client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.914924127Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.914952223Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.915636651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916228353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.91625624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[0]R flush_numbers[0..12]: [213, 225, 225, 218, 206, 221, 223, 226, 221, 201, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916272237Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[0]R generation[0..12]: [107, 114, 114, 110, 104, 111, 113, 115, 111, 102, 114, 114]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916287335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916301833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[1]R flush_numbers[0..12]: [213, 225, 225, 218, 206, 221, 223, 226, 221, 201, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916316191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[1]R generation[0..12]: [107, 114, 114, 110, 104, 111, 113, 115, 111, 102, 114, 114]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916330198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916344436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[2]R flush_numbers[0..12]: [213, 225, 225, 218, 206, 221, 223, 226, 221, 201, 225, 225]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916358134Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[2]R generation[0..12]: [107, 114, 114, 110, 104, 111, 113, 115, 111, 102, 114, 114]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916372002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.91638961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916406157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916419505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916432553Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916445561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"":"downstairs","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916458749Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916478607Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"0","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916498213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"1","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916517701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"client":"2","":"downstairs","session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916531529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916549616Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"6ddf3df0-5952-4c17-8e41-1ee012d1a25a is now active with session: 1c5d5186-2334-44fc-8d78-3bfeab1d245b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916567933Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916582151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647,"session_id":"1c5d5186-2334-44fc-8d78-3bfeab1d245b"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:03.916601378Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 238 len 40960 data: 6 7 7 6 7 6 8 7 6 5 02/30 Read block 452 len 28672 03/30 Read block 441 len 16384 04/30 Read block 46 len 36864 05/30 Read block 359 len 32768 06/30 Flush 07/30 Flush 08/30 Flush 09/30 Write block 215 len 4096 data: 10 10/30 Write block 138 len 36864 data: 7 6 6 7 9 10 11 12 10 11/30 Read block 134 len 28672 12/30 Read block 322 len 32768 13/30 Read block 281 len 8192 14/30 Write block 486 len 4096 data: 14 15/30 Read block 151 len 32768 16/30 Flush 17/30 Write block 586 len 20480 data: 9 7 7 7 10 18/30 Flush 19/30 Write block 109 len 28672 data: 10 9 12 15 17 13 13 20/30 Flush 21/30 Write block 367 len 20480 data: 15 14 13 12 11 22/30 Flush 23/30 Write block 402 len 20480 data: 12 11 12 13 14 24/30 Write block 333 len 36864 data: 9 10 8 11 11 9 9 8 8 25/30 Read block 371 len 36864 26/30 Write block 570 len 28672 data: 10 13 12 11 11 11 11 27/30 Write block 271 len 32768 data: 8 7 10 11 13 13 13 11 28/30 Read block 331 len 16384 29/30 Read block 64 len 32768 30/30 Write block 77 len 40960 data: 8 10 9 8 7 8 9 8 6 5 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:03.945890447Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1647} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 114 114 114 225 225 225 F F F 2 040-059 114 114 114 225 225 225 F F F 3 060-079 110 110 110 218 218 218 T T T 4 080-099 104 104 104 206 206 206 T T T 5 100-119 116 116 116 232 232 232 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 116 116 116 230 230 230 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 116 116 116 230 230 230 F F F 11 220-239 116 116 116 227 227 227 F F F 12 240-259 116 116 116 227 227 227 F F F 13 260-279 106 106 106 212 212 212 T T T 14 280-299 108 108 108 214 214 214 F F F 15 300-319 114 114 114 225 225 225 F F F 16 320-339 100 100 100 196 196 196 T T T 17 340-359 113 113 113 223 223 223 T T T 18 360-379 116 116 116 233 233 233 F F F 19 380-399 115 115 115 226 226 226 F F F 20 400-419 115 115 115 226 226 226 T T T 21 420-439 113 113 113 223 223 223 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 114 114 114 225 225 225 F F F 24 480-499 116 116 116 230 230 230 F F F 25 500-519 115 115 115 226 226 226 F F F 26 520-539 115 115 115 226 226 226 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 115 115 115 226 226 226 T T T 29 580-599 116 116 116 231 231 231 F F F Max gen: 116, Max flush: 233 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 114 114 114 225 225 225 F F F 2 040-059 114 114 114 225 225 225 F F F 3 060-079 117 117 117 234 234 234 F F F 4 080-099 117 117 117 234 234 234 F F F 5 100-119 116 116 116 232 232 232 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 116 116 116 230 230 230 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 116 116 116 230 230 230 F F F 11 220-239 116 116 116 227 227 227 F F F 12 240-259 116 116 116 227 227 227 F F F 13 260-279 117 117 117 234 234 234 F F F 14 280-299 108 108 108 214 214 214 F F F 15 300-319 114 114 114 225 225 225 F F F 16 320-339 117 117 117 234 234 234 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 116 116 116 233 233 233 F F F 19 380-399 115 115 115 226 226 226 F F F 20 400-419 117 117 117 234 234 234 F F F 21 420-439 113 113 113 223 223 223 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 114 114 114 225 225 225 F F F 24 480-499 116 116 116 230 230 230 F F F 25 500-519 115 115 115 226 226 226 F F F 26 520-539 115 115 115 226 226 226 F F F 27 540-559 110 110 110 219 219 219 F F F 28 560-579 117 117 117 234 234 234 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 117, Max flush: 234 Begin loop 59 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.186704018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187467644Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.18749832Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655} {"msg":"Crucible e6b0d612-fa0b-479b-8728-a47f9dac9a9f has session id: d332f673-2d54-48ce-8a7d-2a43ed60d235","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187520727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Upstairs opts: Upstairs UUID: e6b0d612-fa0b-479b-8728-a47f9dac9a9f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187553652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Crucible stats registered with UUID: e6b0d612-fa0b-479b-8728-a47f9dac9a9f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187575819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187808164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655} {"msg":"e6b0d612-fa0b-479b-8728-a47f9dac9a9f active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187848608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187874074Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.18790069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187926047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187948483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.18797198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.187996806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188023512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188051878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188077954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_connection connected from Ok(127.0.0.1:42448)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.1881034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_connection connected from Ok(127.0.0.1:42624)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188343445Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:18.188383399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:18.188402956Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:18.188419354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188435271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188448749Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.188639411Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.189133358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.189165923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.189192679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.189220915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.189797919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.189825225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.190340069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.190367095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.190381993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.19040153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.190415278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:18.190428536Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.190163784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_connection connected from Ok(127.0.0.1:47522)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.190317941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"io task","client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.191280609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.191308225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.191844235Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192408031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192431428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[0]R flush_numbers[0..12]: [213, 225, 225, 234, 234, 232, 230, 230, 221, 201, 230, 227]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192446526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[0]R generation[0..12]: [107, 114, 114, 117, 117, 116, 116, 116, 111, 102, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192461234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192475772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[1]R flush_numbers[0..12]: [213, 225, 225, 234, 234, 232, 230, 230, 221, 201, 230, 227]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192489849Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[1]R generation[0..12]: [107, 114, 114, 117, 117, 116, 116, 116, 111, 102, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192503937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192518295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[2]R flush_numbers[0..12]: [213, 225, 225, 234, 234, 232, 230, 230, 221, 201, 230, 227]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192532393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[2]R generation[0..12]: [107, 114, 114, 117, 117, 116, 116, 116, 111, 102, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192546391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192564728Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192578326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192594924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Next flush: 235","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192608012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.19262114Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"":"downstairs","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192634718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192646726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"0","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192660214Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"1","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192679141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"client":"2","":"downstairs","session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192696019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192714546Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"e6b0d612-fa0b-479b-8728-a47f9dac9a9f is now active with session: d332f673-2d54-48ce-8a7d-2a43ed60d235","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192728424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192740512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655,"session_id":"d332f673-2d54-48ce-8a7d-2a43ed60d235"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:28.192753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 479 len 36864 data: 11 12 11 15 15 17 16 15 13 02/30 Write block 258 len 32768 data: 13 11 8 9 8 7 7 9 03/30 Read block 320 len 40960 04/30 Write block 270 len 32768 data: 7 9 8 11 12 14 14 14 05/30 Read block 187 len 32768 06/30 Write block 550 len 8192 data: 7 5 07/30 Read block 93 len 20480 08/30 Write block 46 len 16384 data: 13 13 9 9 09/30 Flush 10/30 Write block 82 len 12288 data: 9 10 9 11/30 Write block 154 len 24576 data: 7 7 12 13 12 12 12/30 Read block 336 len 24576 13/30 Write block 429 len 40960 data: 11 11 12 13 12 12 9 11 11 11 14/30 Read block 419 len 32768 15/30 Read block 407 len 20480 16/30 Write block 245 len 40960 data: 8 7 6 5 9 10 11 12 13 12 17/30 Write block 528 len 32768 data: 6 5 5 8 8 8 11 10 18/30 Flush 19/30 Write block 32 len 4096 data: 8 20/30 Read block 289 len 32768 21/30 Read block 250 len 12288 22/30 Read block 273 len 4096 23/30 Read block 269 len 20480 24/30 Read block 449 len 32768 25/30 Write block 566 len 40960 data: 9 9 12 10 11 14 13 12 12 12 26/30 Write block 288 len 28672 data: 11 11 12 13 11 8 9 27/30 Write block 396 len 24576 data: 13 10 10 10 12 13 28/30 Write block 508 len 36864 data: 11 14 13 12 12 9 12 12 10 29/30 Read block 8 len 8192 30/30 Read block 392 len 20480 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 1 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false 31 Acked 1030 Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 12 11 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:28.221139123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1655} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 114 114 114 225 225 225 T T T 2 040-059 118 118 118 235 235 235 F F F 3 060-079 117 117 117 234 234 234 F F F 4 080-099 118 118 118 236 236 236 F F F 5 100-119 116 116 116 232 232 232 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 118 118 118 236 236 236 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 116 116 116 230 230 230 F F F 11 220-239 116 116 116 227 227 227 F F F 12 240-259 118 118 118 236 236 236 F F F 13 260-279 118 118 118 235 235 235 F F F 14 280-299 108 108 108 214 214 214 T T T 15 300-319 114 114 114 225 225 225 F F F 16 320-339 117 117 117 234 234 234 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 116 116 116 233 233 233 F F F 19 380-399 115 115 115 226 226 226 T T T 20 400-419 117 117 117 234 234 234 T T T 21 420-439 118 118 118 236 236 236 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 118 118 118 235 235 235 F F F 24 480-499 118 118 118 235 235 235 F F F 25 500-519 115 115 115 226 226 226 T T T 26 520-539 118 118 118 237 237 237 F F F 27 540-559 118 118 118 235 235 235 F F F 28 560-579 117 117 117 234 234 234 T T T 29 580-599 116 116 116 231 231 231 F F F Max gen: 118, Max flush: 237 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 F F F 1 020-039 119 119 119 238 238 238 F F F 2 040-059 118 118 118 235 235 235 F F F 3 060-079 117 117 117 234 234 234 F F F 4 080-099 118 118 118 236 236 236 F F F 5 100-119 116 116 116 232 232 232 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 118 118 118 236 236 236 F F F 8 160-179 111 111 111 221 221 221 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 116 116 116 230 230 230 F F F 11 220-239 116 116 116 227 227 227 F F F 12 240-259 118 118 118 236 236 236 F F F 13 260-279 118 118 118 235 235 235 F F F 14 280-299 119 119 119 238 238 238 F F F 15 300-319 114 114 114 225 225 225 F F F 16 320-339 117 117 117 234 234 234 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 116 116 116 233 233 233 F F F 19 380-399 119 119 119 238 238 238 F F F 20 400-419 119 119 119 238 238 238 F F F 21 420-439 118 118 118 236 236 236 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 118 118 118 235 235 235 F F F 24 480-499 118 118 118 235 235 235 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 118 118 118 237 237 237 F F F 27 540-559 118 118 118 235 235 235 F F F 28 560-579 119 119 119 238 238 238 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 119, Max flush: 238 Begin loop 60 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.454125404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.454621751Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.454653296Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663} {"msg":"Crucible 560e9797-4d82-4dd3-97c6-951086c0e4c3 has session id: 69fade80-cf45-4356-8b60-c06961ff80d8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.454979958Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Upstairs opts: Upstairs UUID: 560e9797-4d82-4dd3-97c6-951086c0e4c3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455042478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Crucible stats registered with UUID: 560e9797-4d82-4dd3-97c6-951086c0e4c3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455064405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455449308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663} {"msg":"560e9797-4d82-4dd3-97c6-951086c0e4c3 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455523747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455541384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455556012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.45556961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455583608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455604085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455629561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455651118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455665716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.45570824Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:42.455832181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:42.455871675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61268)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455895752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:42.455929647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:65136)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455952034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.45597596Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.455997077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.456164452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.456669837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.456695923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.456710701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.456724679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.45732737Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.457354835Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.457829185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.457855611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.457870219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.457899085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.457916822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:42.45793027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.457386001Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_connection connected from Ok(127.0.0.1:55525)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.457464719Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"io task","client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.458500166Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.458527762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459097608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459681231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459708817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[0]R flush_numbers[0..12]: [213, 238, 235, 234, 236, 232, 230, 236, 221, 201, 230, 227]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459725384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[0]R generation[0..12]: [107, 119, 118, 117, 118, 116, 116, 118, 111, 102, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459740112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.45975453Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[1]R flush_numbers[0..12]: [213, 238, 235, 234, 236, 232, 230, 236, 221, 201, 230, 227]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459768618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[1]R generation[0..12]: [107, 119, 118, 117, 118, 116, 116, 118, 111, 102, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459782396Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459796384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[2]R flush_numbers[0..12]: [213, 238, 235, 234, 236, 232, 230, 236, 221, 201, 230, 227]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459814651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[2]R generation[0..12]: [107, 119, 118, 117, 118, 116, 116, 118, 111, 102, 116, 116]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459828449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459842397Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459858915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459871903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459884701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459897379Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"":"downstairs","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459910397Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459922195Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"0","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459935253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"1","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459948261Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"client":"2","":"downstairs","session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459961659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459974018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"560e9797-4d82-4dd3-97c6-951086c0e4c3 is now active with session: 69fade80-cf45-4356-8b60-c06961ff80d8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459986626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.459998304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663,"session_id":"69fade80-cf45-4356-8b60-c06961ff80d8"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:32:52.460012642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 306 len 20480 data: 8 8 6 7 10 02/30 Write block 238 len 16384 data: 7 8 8 7 03/30 Write block 476 len 32768 data: 11 9 11 12 13 12 16 16 04/30 Read block 257 len 8192 05/30 Read block 461 len 20480 06/30 Read block 557 len 40960 07/30 Write block 96 len 16384 data: 6 5 5 4 08/30 Read block 342 len 4096 09/30 Write block 376 len 32768 data: 9 11 12 13 15 12 9 7 10/30 Read block 474 len 40960 11/30 Read block 326 len 16384 12/30 Read block 15 len 8192 13/30 Read block 477 len 20480 14/30 Write block 566 len 24576 data: 10 10 13 11 12 15 15/30 Read block 125 len 8192 16/30 Read block 242 len 4096 17/30 Read block 428 len 8192 18/30 Write block 437 len 12288 data: 12 12 10 19/30 Write block 571 len 20480 data: 16 14 13 13 13 20/30 Read block 411 len 36864 21/30 Read block 50 len 36864 22/30 Write block 273 len 28672 data: 12 13 15 15 15 12 9 23/30 Flush 24/30 Write block 329 len 24576 data: 8 8 8 8 10 11 25/30 Write block 5 len 4096 data: 8 26/30 Read block 210 len 24576 27/30 Write block 63 len 40960 data: 11 13 12 10 10 8 11 5 3 4 28/30 Read block 432 len 20480 29/30 Write block 159 len 20480 data: 13 15 13 16 15 30/30 Read block 373 len 20480 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 5 Done Done Sent false 31 Acked 1030 Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 2 3 Done 7 6 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:32:52.485031423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1663} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 107 107 107 213 213 213 T T T 1 020-039 119 119 119 238 238 238 F F F 2 040-059 118 118 118 235 235 235 F F F 3 060-079 117 117 117 234 234 234 T T T 4 080-099 120 120 120 239 239 239 F F F 5 100-119 116 116 116 232 232 232 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 118 118 118 236 236 236 T T T 8 160-179 111 111 111 221 221 221 T T T 9 180-199 102 102 102 201 201 201 F F F 10 200-219 116 116 116 230 230 230 F F F 11 220-239 120 120 120 239 239 239 F F F 12 240-259 120 120 120 239 239 239 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 119 119 119 238 238 238 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 117 117 117 234 234 234 T T T 17 340-359 117 117 117 234 234 234 F F F 18 360-379 120 120 120 239 239 239 F F F 19 380-399 120 120 120 239 239 239 F F F 20 400-419 119 119 119 238 238 238 F F F 21 420-439 120 120 120 239 239 239 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 120 120 120 239 239 239 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 118 118 118 237 237 237 F F F 27 540-559 118 118 118 235 235 235 F F F 28 560-579 120 120 120 239 239 239 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 120, Max flush: 240 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 241 241 241 F F F 1 020-039 119 119 119 238 238 238 F F F 2 040-059 118 118 118 235 235 235 F F F 3 060-079 121 121 121 241 241 241 F F F 4 080-099 120 120 120 239 239 239 F F F 5 100-119 116 116 116 232 232 232 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 121 121 121 241 241 241 F F F 8 160-179 121 121 121 241 241 241 F F F 9 180-199 102 102 102 201 201 201 F F F 10 200-219 116 116 116 230 230 230 F F F 11 220-239 120 120 120 239 239 239 F F F 12 240-259 120 120 120 239 239 239 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 119 119 119 238 238 238 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 120 120 120 239 239 239 F F F 19 380-399 120 120 120 239 239 239 F F F 20 400-419 119 119 119 238 238 238 F F F 21 420-439 120 120 120 239 239 239 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 120 120 120 239 239 239 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 118 118 118 237 237 237 F F F 27 540-559 118 118 118 235 235 235 F F F 28 560-579 120 120 120 239 239 239 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 121, Max flush: 241 Begin loop 61 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.723752484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724168202Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724199727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671} {"msg":"Crucible 1e758fa0-e0b4-45db-ab01-c95d617a9bc8 has session id: 22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724333058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Upstairs opts: Upstairs UUID: 1e758fa0-e0b4-45db-ab01-c95d617a9bc8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724469657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Crucible stats registered with UUID: 1e758fa0-e0b4-45db-ab01-c95d617a9bc8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724496993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724831874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671} {"msg":"1e758fa0-e0b4-45db-ab01-c95d617a9bc8 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.72485715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724871458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724886666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724900313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724913942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724929239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724953456Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724972193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724986141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.724999719Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:06.725143157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:40222)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.725179202Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:06.725208178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:06.725242823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.725366904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.725384602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:60498)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.72539857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.725412478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.725965696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.72600369Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.726033685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.726060102Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.726616179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.726644205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.727138382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.727164568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.727179046Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.727207462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.727225089Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:06.727238617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.726377604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41896)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.726614329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"io task","client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.727578666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.72761765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728187776Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728773809Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728801455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[0]R flush_numbers[0..12]: [241, 238, 235, 241, 239, 232, 230, 241, 241, 201, 230, 239]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728818012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[0]R generation[0..12]: [121, 119, 118, 121, 120, 116, 116, 121, 121, 102, 116, 120]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.72883295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728847518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[1]R flush_numbers[0..12]: [241, 238, 235, 241, 239, 232, 230, 241, 241, 201, 230, 239]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728861166Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[1]R generation[0..12]: [121, 119, 118, 121, 120, 116, 116, 121, 121, 102, 116, 120]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728875174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728893901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[2]R flush_numbers[0..12]: [241, 238, 235, 241, 239, 232, 230, 241, 241, 201, 230, 239]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728907959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[2]R generation[0..12]: [121, 119, 118, 121, 120, 116, 116, 121, 121, 102, 116, 120]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728921587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728935535Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728952262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.72896589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728978848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.728992066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"":"downstairs","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729005615Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729017573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"0","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729030911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"1","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729058287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"client":"2","":"downstairs","session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729072394Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729084463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"1e758fa0-e0b4-45db-ab01-c95d617a9bc8 is now active with session: 22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729097211Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729111539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671,"session_id":"22bb8b6c-aeda-4a07-8852-4b1f27bbdb8f"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:16.729123757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 22 len 12288 data: 9 11 10 03/30 Read block 511 len 28672 04/30 Read block 379 len 20480 05/30 Read block 22 len 40960 06/30 Read block 74 len 20480 07/30 Write block 405 len 40960 data: 14 15 11 9 11 11 13 11 9 7 08/30 Read block 75 len 40960 09/30 Write block 251 len 36864 data: 12 13 14 13 8 10 11 14 12 10/30 Read block 476 len 40960 11/30 Write block 79 len 24576 data: 10 9 8 10 11 10 12/30 Write block 95 len 40960 data: 8 7 6 6 5 4 5 5 5 6 13/30 Write block 36 len 8192 data: 3 3 14/30 Flush 15/30 Write block 93 len 40960 data: 9 7 9 8 7 7 6 5 6 6 16/30 Read block 408 len 24576 17/30 Write block 228 len 12288 data: 5 6 5 18/30 Write block 543 len 24576 data: 6 5 6 6 6 6 19/30 Read block 175 len 8192 20/30 Write block 216 len 16384 data: 10 10 9 11 21/30 Read block 105 len 36864 22/30 Flush 23/30 Write block 289 len 24576 data: 12 13 14 12 9 10 24/30 Write block 112 len 24576 data: 16 18 14 14 12 8 25/30 Write block 192 len 24576 data: 7 6 5 8 6 6 26/30 Write block 525 len 36864 data: 7 6 7 7 6 6 9 9 9 27/30 Read block 469 len 4096 28/30 Read block 145 len 4096 29/30 Write block 145 len 36864 data: 13 11 8 8 8 7 7 5 7 30/30 Read block 503 len 16384 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 6 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 9 Done Done Done false 31 Acked 1030 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:16.756707218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1671} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 241 241 241 F F F 1 020-039 122 122 122 243 243 243 F F F 2 040-059 118 118 118 235 235 235 F F F 3 060-079 122 122 122 243 243 243 F F F 4 080-099 122 122 122 244 244 244 F F F 5 100-119 122 122 122 244 244 244 T T T 6 120-139 116 116 116 230 230 230 F F F 7 140-159 121 121 121 241 241 241 T T T 8 160-179 121 121 121 241 241 241 F F F 9 180-199 102 102 102 201 201 201 T T T 10 200-219 122 122 122 245 245 245 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 119 119 119 238 238 238 T T T 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 120 120 120 239 239 239 F F F 19 380-399 120 120 120 239 239 239 F F F 20 400-419 122 122 122 243 243 243 F F F 21 420-439 120 120 120 239 239 239 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 120 120 120 239 239 239 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 118 118 118 237 237 237 T T T 27 540-559 122 122 122 244 244 244 F F F 28 560-579 120 120 120 239 239 239 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 122, Max flush: 245 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 241 241 241 F F F 1 020-039 122 122 122 243 243 243 F F F 2 040-059 118 118 118 235 235 235 F F F 3 060-079 122 122 122 243 243 243 F F F 4 080-099 122 122 122 244 244 244 F F F 5 100-119 123 123 123 246 246 246 F F F 6 120-139 116 116 116 230 230 230 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 121 121 121 241 241 241 F F F 9 180-199 123 123 123 246 246 246 F F F 10 200-219 122 122 122 245 245 245 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 120 120 120 239 239 239 F F F 19 380-399 120 120 120 239 239 239 F F F 20 400-419 122 122 122 243 243 243 F F F 21 420-439 120 120 120 239 239 239 F F F 22 440-459 108 108 108 214 214 214 F F F 23 460-479 120 120 120 239 239 239 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 123 123 123 246 246 246 F F F 27 540-559 122 122 122 244 244 244 F F F 28 560-579 120 120 120 239 239 239 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 123, Max flush: 246 Begin loop 62 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.973720697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974497252Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974518248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679} {"msg":"Crucible 199f1af9-1723-4273-a889-cf263539c21a has session id: b2c620d1-5244-4eec-9b58-863d9d7ba1f7","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974532106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Upstairs opts: Upstairs UUID: 199f1af9-1723-4273-a889-cf263539c21a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974555543Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Crucible stats registered with UUID: 199f1af9-1723-4273-a889-cf263539c21a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974569041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.97491674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679} {"msg":"199f1af9-1723-4273-a889-cf263539c21a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974947865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.974972261Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975017605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.97504611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975072376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975098853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975126438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975152585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.97517917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975201847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:30.975280195Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:30.975314921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34335)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975347506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:30.975374312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975484905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975512651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33246)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975540127Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.975579071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.976120891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.976145857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.976160025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.976173513Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.976702555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.976825266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.97727066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.977308545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.97734268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.977369786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.977397732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:30.977415779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.975810767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:46126)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.975945417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"io task","client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.976980433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.977015378Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.97768057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978227598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978253894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[0]R flush_numbers[0..12]: [241, 243, 235, 243, 244, 246, 230, 246, 241, 246, 245, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978270132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[0]R generation[0..12]: [121, 122, 118, 122, 122, 123, 116, 123, 121, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.97828502Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978299548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[1]R flush_numbers[0..12]: [241, 243, 235, 243, 244, 246, 230, 246, 241, 246, 245, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978313505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[1]R generation[0..12]: [121, 122, 118, 122, 122, 123, 116, 123, 121, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978331273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978345671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[2]R flush_numbers[0..12]: [241, 243, 235, 243, 244, 246, 230, 246, 241, 246, 245, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978359499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[2]R generation[0..12]: [121, 122, 118, 122, 122, 123, 116, 123, 121, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978373097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978386925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:33:40.978399903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.97841531Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Next flush: 247","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978428239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978440897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"":"downstairs","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978453955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978465903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"0","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978479161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"1","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978492109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"client":"2","":"downstairs","session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978504987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978517106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"199f1af9-1723-4273-a889-cf263539c21a is now active with session: b2c620d1-5244-4eec-9b58-863d9d7ba1f7","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978531873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.978543802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679,"session_id":"b2c620d1-5244-4eec-9b58-863d9d7ba1f7"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:40.97855592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 402 len 16384 data: 13 12 13 15 03/30 Read block 350 len 20480 04/30 Read block 117 len 8192 05/30 Read block 500 len 28672 06/30 Write block 555 len 36864 data: 9 12 9 9 7 8 9 9 12 07/30 Write block 33 len 12288 data: 8 4 3 08/30 Read block 351 len 4096 09/30 Write block 397 len 4096 data: 11 10/30 Read block 430 len 8192 11/30 Read block 469 len 24576 12/30 Read block 368 len 8192 13/30 Read block 494 len 16384 14/30 Read block 429 len 4096 15/30 Write block 26 len 28672 data: 7 9 9 9 7 8 9 16/30 Flush 17/30 Write block 40 len 32768 data: 4 7 10 12 16 13 14 14 18/30 Read block 8 len 8192 19/30 Write block 400 len 16384 data: 13 14 14 13 20/30 Flush 21/30 Write block 116 len 40960 data: 13 9 10 9 9 8 8 8 6 7 22/30 Write block 443 len 16384 data: 11 9 9 8 23/30 Read block 73 len 8192 24/30 Read block 553 len 20480 25/30 Write block 175 len 4096 data: 9 26/30 Read block 179 len 8192 27/30 Read block 195 len 24576 28/30 Read block 543 len 16384 29/30 Read block 458 len 36864 30/30 Read block 278 len 32768 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 5 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Read 6 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:41.004975713Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1679} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 241 241 241 F F F 1 020-039 124 124 124 248 248 248 F F F 2 040-059 124 124 124 249 249 249 F F F 3 060-079 122 122 122 243 243 243 F F F 4 080-099 122 122 122 244 244 244 F F F 5 100-119 124 124 124 250 250 250 F F F 6 120-139 124 124 124 250 250 250 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 121 121 121 241 241 241 T T T 9 180-199 123 123 123 246 246 246 F F F 10 200-219 122 122 122 245 245 245 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 120 120 120 239 239 239 F F F 19 380-399 124 124 124 248 248 248 F F F 20 400-419 124 124 124 249 249 249 F F F 21 420-439 120 120 120 239 239 239 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 120 120 120 239 239 239 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 123 123 123 246 246 246 F F F 27 540-559 124 124 124 248 248 248 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 124, Max flush: 250 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 241 241 241 F F F 1 020-039 124 124 124 248 248 248 F F F 2 040-059 124 124 124 249 249 249 F F F 3 060-079 122 122 122 243 243 243 F F F 4 080-099 122 122 122 244 244 244 F F F 5 100-119 124 124 124 250 250 250 F F F 6 120-139 124 124 124 250 250 250 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 125 125 125 251 251 251 F F F 9 180-199 123 123 123 246 246 246 F F F 10 200-219 122 122 122 245 245 245 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 120 120 120 239 239 239 F F F 19 380-399 124 124 124 248 248 248 F F F 20 400-419 124 124 124 249 249 249 F F F 21 420-439 120 120 120 239 239 239 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 120 120 120 239 239 239 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 F F F 26 520-539 123 123 123 246 246 246 F F F 27 540-559 124 124 124 248 248 248 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 125, Max flush: 251 Begin loop 63 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.174345439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.174747159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.174765486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687} {"msg":"Crucible e47e7bd8-e49e-4acc-a78f-b2ccf7897b1a has session id: 1df21eec-dac5-4b19-a9ef-1ea7bbf26c26","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.174931322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Upstairs opts: Upstairs UUID: e47e7bd8-e49e-4acc-a78f-b2ccf7897b1a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.174979335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Crucible stats registered with UUID: e47e7bd8-e49e-4acc-a78f-b2ccf7897b1a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175020189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175402662Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687} {"msg":"e47e7bd8-e49e-4acc-a78f-b2ccf7897b1a active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175556569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175577226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175592414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175606212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.17561962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175633028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175646356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175701538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175719995Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.175733433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_connection connected from Ok(127.0.0.1:54732)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176069273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_connection connected from Ok(127.0.0.1:64397)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176108387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:51.176144932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:51.176177087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:33:51.176334144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176352211Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176365909Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176481212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176865135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176903489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176933185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.176960441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.177581599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.177621993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.178119789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.178144416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.178162573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.178190559Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.178204797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:33:51.178217905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.177744514Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_connection connected from Ok(127.0.0.1:60794)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.177900301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"io task","client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.178823574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.178857609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.179354865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.17992988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.179952067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[0]R flush_numbers[0..12]: [241, 248, 249, 243, 244, 250, 250, 246, 251, 246, 245, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.179966994Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[0]R generation[0..12]: [121, 124, 124, 122, 122, 124, 124, 123, 125, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.179981222Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.17999535Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[1]R flush_numbers[0..12]: [241, 248, 249, 243, 244, 250, 250, 246, 251, 246, 245, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180009428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[1]R generation[0..12]: [121, 124, 124, 122, 122, 124, 124, 123, 125, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180028385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180043233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[2]R flush_numbers[0..12]: [241, 248, 249, 243, 244, 250, 250, 246, 251, 246, 245, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180057161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[2]R generation[0..12]: [121, 124, 124, 122, 122, 124, 124, 123, 125, 123, 122, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180070959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180085057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180097945Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180111223Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180124221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180141319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"":"downstairs","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180154777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180166635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"0","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180179853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"1","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180193011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"client":"2","":"downstairs","session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180206129Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180218307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"e47e7bd8-e49e-4acc-a78f-b2ccf7897b1a is now active with session: 1df21eec-dac5-4b19-a9ef-1ea7bbf26c26","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180233265Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180245263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687,"session_id":"1df21eec-dac5-4b19-a9ef-1ea7bbf26c26"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:01.180257371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 207 len 24576 data: 17 14 11 11 13 11 02/30 Read block 444 len 8192 03/30 Read block 244 len 12288 04/30 Read block 35 len 32768 05/30 Read block 162 len 32768 06/30 Write block 164 len 36864 data: 15 15 14 14 12 14 12 11 9 07/30 Read block 521 len 4096 08/30 Write block 414 len 32768 data: 8 7 9 7 5 5 7 7 09/30 Read block 129 len 4096 10/30 Write block 118 len 32768 data: 11 10 10 9 9 9 7 8 11/30 Write block 402 len 32768 data: 15 14 14 16 16 12 10 12 12/30 Read block 192 len 12288 13/30 Write block 477 len 8192 data: 10 12 14/30 Read block 383 len 28672 15/30 Read block 143 len 8192 16/30 Flush 17/30 Write block 379 len 16384 data: 14 16 13 10 18/30 Read block 445 len 4096 19/30 Read block 252 len 8192 20/30 Flush 21/30 Write block 6 len 28672 data: 8 8 8 11 12 12 10 22/30 Read block 363 len 20480 23/30 Read block 369 len 12288 24/30 Read block 319 len 8192 25/30 Read block 453 len 32768 26/30 Read block 255 len 28672 27/30 Write block 508 len 36864 data: 12 15 14 13 13 10 13 13 11 28/30 Write block 79 len 36864 data: 11 10 9 11 12 11 7 6 5 29/30 Read block 393 len 24576 30/30 Read block 439 len 40960 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 7 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Read 2 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 7 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 9 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:01.206867657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1687} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 241 241 241 T T T 1 020-039 124 124 124 248 248 248 F F F 2 040-059 124 124 124 249 249 249 F F F 3 060-079 122 122 122 243 243 243 T T T 4 080-099 122 122 122 244 244 244 T T T 5 100-119 126 126 126 252 252 252 F F F 6 120-139 126 126 126 252 252 252 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 126 126 126 252 252 252 F F F 9 180-199 123 123 123 246 246 246 F F F 10 200-219 126 126 126 252 252 252 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 126 126 126 253 253 253 F F F 19 380-399 126 126 126 253 253 253 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 126 126 126 252 252 252 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 119 119 119 238 238 238 T T T 26 520-539 123 123 123 246 246 246 F F F 27 540-559 124 124 124 248 248 248 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 126, Max flush: 253 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 124 124 124 248 248 248 F F F 2 040-059 124 124 124 249 249 249 F F F 3 060-079 127 127 127 254 254 254 F F F 4 080-099 127 127 127 254 254 254 F F F 5 100-119 126 126 126 252 252 252 F F F 6 120-139 126 126 126 252 252 252 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 126 126 126 252 252 252 F F F 9 180-199 123 123 123 246 246 246 F F F 10 200-219 126 126 126 252 252 252 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 121 121 121 241 241 241 F F F 17 340-359 117 117 117 234 234 234 F F F 18 360-379 126 126 126 253 253 253 F F F 19 380-399 126 126 126 253 253 253 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 126 126 126 252 252 252 F F F 24 480-499 120 120 120 239 239 239 F F F 25 500-519 127 127 127 254 254 254 F F F 26 520-539 123 123 123 246 246 246 F F F 27 540-559 124 124 124 248 248 248 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 127, Max flush: 254 Begin loop 64 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.412327799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.412709442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.412743157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695} {"msg":"Crucible f6a21075-8839-40a8-9dc7-f26da9efa68e has session id: 0597dc3e-59ac-465a-b436-fcd3a8915022","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.41319394Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Upstairs opts: Upstairs UUID: f6a21075-8839-40a8-9dc7-f26da9efa68e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413229705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Crucible stats registered with UUID: f6a21075-8839-40a8-9dc7-f26da9efa68e","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413245403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413647723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695} {"msg":"f6a21075-8839-40a8-9dc7-f26da9efa68e active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413722162Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.4137385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413752917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413766355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413779733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413796981Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413821927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413837505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413851393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.413864921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_connection connected from Ok(127.0.0.1:38826)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.414168456Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:15.414318504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:15.414358998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_connection connected from Ok(127.0.0.1:56116)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.414378035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:15.414394602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.414406951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.414419959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.414584594Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.41501864Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.415044466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.415168758Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.415189775Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.415572368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.415679292Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.416027191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.416060226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.416086942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.416190936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.416209563Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:15.416222731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.41508679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_connection connected from Ok(127.0.0.1:64761)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.41521833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"io task","client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.416217482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.416245468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.416822362Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417351974Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.41738019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[0]R flush_numbers[0..12]: [254, 248, 249, 254, 254, 252, 252, 246, 252, 246, 252, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417395927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[0]R generation[0..12]: [127, 124, 124, 127, 127, 126, 126, 123, 126, 123, 126, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417410485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417424393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[1]R flush_numbers[0..12]: [254, 248, 249, 254, 254, 252, 252, 246, 252, 246, 252, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417442301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[1]R generation[0..12]: [127, 124, 124, 127, 127, 126, 126, 123, 126, 123, 126, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417456328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417470216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[2]R flush_numbers[0..12]: [254, 248, 249, 254, 254, 252, 252, 246, 252, 246, 252, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417484124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[2]R generation[0..12]: [127, 124, 124, 127, 127, 126, 126, 123, 126, 123, 126, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417497722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.41751167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Max found gen is 128","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417524538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417537896Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417550714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417563463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.41758105Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417593068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"0","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417606176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"1","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417619144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"client":"2","":"downstairs","session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417632002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.41764581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"f6a21075-8839-40a8-9dc7-f26da9efa68e is now active with session: 0597dc3e-59ac-465a-b436-fcd3a8915022","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417658718Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417670577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695,"session_id":"0597dc3e-59ac-465a-b436-fcd3a8915022"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:25.417682745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 509 len 40960 data: 16 15 14 14 11 14 14 12 9 9 02/30 Read block 515 len 8192 03/30 Read block 530 len 16384 04/30 Read block 281 len 4096 05/30 Read block 62 len 20480 06/30 Write block 380 len 40960 data: 17 14 11 8 8 7 8 7 10 12 07/30 Write block 98 len 24576 data: 8 7 6 7 7 6 08/30 Write block 363 len 24576 data: 9 10 11 13 16 15 09/30 Read block 198 len 32768 10/30 Read block 132 len 8192 11/30 Read block 65 len 28672 12/30 Write block 333 len 36864 data: 11 12 9 12 12 10 10 9 9 13/30 Read block 385 len 8192 14/30 Flush 15/30 Write block 57 len 40960 data: 11 10 9 13 12 12 12 14 13 11 16/30 Read block 201 len 8192 17/30 Write block 109 len 24576 data: 11 10 13 17 19 15 18/30 Write block 375 len 12288 data: 13 10 12 19/30 Read block 262 len 40960 20/30 Write block 497 len 16384 data: 17 15 13 13 21/30 Write block 180 len 20480 data: 7 8 6 7 6 22/30 Read block 204 len 40960 23/30 Flush 24/30 Flush 25/30 Write block 167 len 36864 data: 15 13 15 13 12 10 10 9 10 26/30 Write block 209 len 36864 data: 12 12 14 12 9 9 11 11 11 27/30 Write block 551 len 8192 data: 6 4 28/30 Read block 120 len 32768 29/30 Read block 211 len 4096 30/30 Read block 475 len 4096 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 1 Done Done Done false 31 Acked 1030 Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 6 5 6 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:25.443651266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1695} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 124 124 124 248 248 248 F F F 2 040-059 128 128 128 256 256 256 F F F 3 060-079 128 128 128 256 256 256 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 128 128 128 256 256 256 F F F 6 120-139 126 126 126 252 252 252 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 126 126 126 252 252 252 T T T 9 180-199 128 128 128 256 256 256 F F F 10 200-219 126 126 126 252 252 252 T T T 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 128 128 128 255 255 255 F F F 17 340-359 128 128 128 255 255 255 F F F 18 360-379 128 128 128 256 256 256 F F F 19 380-399 128 128 128 255 255 255 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 126 126 126 252 252 252 F F F 24 480-499 128 128 128 256 256 256 F F F 25 500-519 128 128 128 256 256 256 F F F 26 520-539 123 123 123 246 246 246 F F F 27 540-559 124 124 124 248 248 248 T T T 28 560-579 124 124 124 248 248 248 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 128, Max flush: 256 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 124 124 124 248 248 248 F F F 2 040-059 128 128 128 256 256 256 F F F 3 060-079 128 128 128 256 256 256 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 128 128 128 256 256 256 F F F 6 120-139 126 126 126 252 252 252 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 129 129 129 257 257 257 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 F F F 16 320-339 128 128 128 255 255 255 F F F 17 340-359 128 128 128 255 255 255 F F F 18 360-379 128 128 128 256 256 256 F F F 19 380-399 128 128 128 255 255 255 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 126 126 126 252 252 252 F F F 24 480-499 128 128 128 256 256 256 F F F 25 500-519 128 128 128 256 256 256 F F F 26 520-539 123 123 123 246 246 246 F F F 27 540-559 129 129 129 257 257 257 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 116 116 116 231 231 231 F F F Max gen: 129, Max flush: 257 Begin loop 65 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.635301744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.635888107Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.635927112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703} {"msg":"Crucible bac031ee-fa9a-4f3f-b8f7-2580f2de8e15 has session id: e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.635949688Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Upstairs opts: Upstairs UUID: bac031ee-fa9a-4f3f-b8f7-2580f2de8e15, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.635982164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Crucible stats registered with UUID: bac031ee-fa9a-4f3f-b8f7-2580f2de8e15","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.635998491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.63648041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703} {"msg":"bac031ee-fa9a-4f3f-b8f7-2580f2de8e15 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636586324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.63661089Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636626938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636640806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636654314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636667552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.63668086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636712245Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636728863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636742641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_connection connected from Ok(127.0.0.1:53310)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636910646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_connection connected from Ok(127.0.0.1:51584)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.636990984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:39.637142881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:39.637178126Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:39.637204262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637224619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637246526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637362069Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637900259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637928355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637943333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.637956651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.638504999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.638540504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.639083274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.63911061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.639127157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.639140036Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.639154103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:39.639167251Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.638552752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_connection connected from Ok(127.0.0.1:46010)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.638768851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"io task","client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.639657379Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.639685145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.640295234Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.640908943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.64092806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[0]R flush_numbers[0..12]: [254, 248, 256, 256, 255, 256, 252, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.640942568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[0]R generation[0..12]: [127, 124, 128, 128, 128, 128, 126, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.640956576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.640970734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[1]R flush_numbers[0..12]: [254, 248, 256, 256, 255, 256, 252, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.640989911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[1]R generation[0..12]: [127, 124, 128, 128, 128, 128, 126, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641016817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641031755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[2]R flush_numbers[0..12]: [254, 248, 256, 256, 255, 256, 252, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641045623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[2]R generation[0..12]: [127, 124, 128, 128, 128, 128, 126, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641059291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641073269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641086147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"timeWait for a query_work_queue command to finish before sending IO ":"2024-08-05T19:34:49.641099005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641115803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641128591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"":"downstairs","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641141409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641153187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"0","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641166085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"1","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641179173Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"client":"2","":"downstairs","session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641194461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641206709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"bac031ee-fa9a-4f3f-b8f7-2580f2de8e15 is now active with session: e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641219267Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641236095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703,"session_id":"e2bd04f3-9e26-439a-8b3d-e1b4cbcc4715"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:34:49.641256972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 4096 data: 15 02/30 Read block 510 len 36864 03/30 Write block 33 len 32768 data: 9 5 4 4 4 3 3 5 04/30 Read block 88 len 24576 05/30 Write block 580 len 24576 data: 13 14 9 6 6 8 06/30 Write block 464 len 36864 data: 5 7 9 13 9 6 6 5 5 07/30 Write block 122 len 8192 data: 10 10 08/30 Flush 09/30 Flush 10/30 Write block 484 len 32768 data: 18 17 16 14 16 16 13 10 11/30 Read block 539 len 20480 12/30 Read block 269 len 12288 13/30 Write block 595 len 12288 data: 10 8 7 14/30 Read block 397 len 40960 15/30 Write block 28 len 16384 data: 10 10 8 9 16/30 Write block 103 len 20480 data: 7 7 8 10 9 17/30 Read block 211 len 20480 18/30 Write block 395 len 16384 data: 10 14 12 11 19/30 Flush 20/30 Flush 21/30 Write block 388 len 20480 data: 11 13 13 11 10 22/30 Flush 23/30 Write block 501 len 32768 data: 13 11 13 12 12 12 11 13 24/30 Read block 111 len 36864 25/30 Write block 518 len 40960 data: 10 8 9 9 8 9 9 8 7 8 26/30 Read block 35 len 40960 27/30 Read block 310 len 16384 28/30 Write block 319 len 8192 data: 6 6 29/30 Write block 391 len 28672 data: 12 11 10 11 11 15 13 30/30 Write block 311 len 12288 data: 11 10 10 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:34:49.669147857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1703} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 130 130 130 260 260 260 F F F 2 040-059 130 130 130 258 258 258 F F F 3 060-079 128 128 128 256 256 256 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 130 130 130 260 260 260 F F F 6 120-139 130 130 130 258 258 258 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 129 129 129 257 257 257 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 120 120 120 239 239 239 T T T 16 320-339 128 128 128 255 255 255 T T T 17 340-359 128 128 128 255 255 255 F F F 18 360-379 130 130 130 258 258 258 F F F 19 380-399 130 130 130 263 263 263 T T T 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 130 130 130 258 258 258 F F F 24 480-499 130 130 130 260 260 260 F F F 25 500-519 128 128 128 256 256 256 T T T 26 520-539 123 123 123 246 246 246 T T T 27 540-559 129 129 129 257 257 257 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 130 130 130 260 260 260 F F F Max gen: 130, Max flush: 263 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 130 130 130 260 260 260 F F F 2 040-059 130 130 130 258 258 258 F F F 3 060-079 128 128 128 256 256 256 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 130 130 130 260 260 260 F F F 6 120-139 130 130 130 258 258 258 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 129 129 129 257 257 257 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 131 131 131 264 264 264 F F F 16 320-339 131 131 131 264 264 264 F F F 17 340-359 128 128 128 255 255 255 F F F 18 360-379 130 130 130 258 258 258 F F F 19 380-399 131 131 131 264 264 264 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 130 130 130 258 258 258 F F F 24 480-499 130 130 130 260 260 260 F F F 25 500-519 131 131 131 264 264 264 F F F 26 520-539 131 131 131 264 264 264 F F F 27 540-559 129 129 129 257 257 257 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 130 130 130 260 260 260 F F F Max gen: 131, Max flush: 264 Begin loop 66 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.888981248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.889546754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.889584169Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711} {"msg":"Crucible bc0a8d37-7935-467f-b3bc-66be756d05b8 has session id: 8d54f235-e599-4c4c-bfca-1a165ab8f55b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.889606655Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Upstairs opts: Upstairs UUID: bc0a8d37-7935-467f-b3bc-66be756d05b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.88963978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Crucible stats registered with UUID: bc0a8d37-7935-467f-b3bc-66be756d05b8","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.889662997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.889983189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711} {"msg":"bc0a8d37-7935-467f-b3bc-66be756d05b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890108871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890126538Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890141266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890154974Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890168282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.89018127Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890194618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890211865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890225514Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890242851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:03.890422324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_connection connected from Ok(127.0.0.1:55734)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890461508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:03.890508141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:03.890593349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890618825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890638552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_connection connected from Ok(127.0.0.1:64068)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890661099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.890685195Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.891297145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.89133179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.891359086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.891421566Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.891823097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.891931661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.892391042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.892416549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.892430747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.892442905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.892456533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:03.892469791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.892681288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_connection connected from Ok(127.0.0.1:56260)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.89280268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"io task","client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.893706816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.893734502Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894330373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894870943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894898609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[0]R flush_numbers[0..12]: [254, 260, 258, 256, 255, 260, 258, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894915627Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[0]R generation[0..12]: [127, 130, 130, 128, 128, 130, 130, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894930124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894948662Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[1]R flush_numbers[0..12]: [254, 260, 258, 256, 255, 260, 258, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.89496297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[1]R generation[0..12]: [127, 130, 130, 128, 128, 130, 130, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894977077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.894991185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[2]R flush_numbers[0..12]: [254, 260, 258, 256, 255, 260, 258, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895005473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[2]R generation[0..12]: [127, 130, 130, 128, 128, 130, 130, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895019361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:35:13.895033179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895047587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895060915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895073573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895086181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"":"downstairs","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895099159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895110848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"0","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895123986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"1","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895139013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"client":"2","":"downstairs","session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895152171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.89516418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"bc0a8d37-7935-467f-b3bc-66be756d05b8 is now active with session: 8d54f235-e599-4c4c-bfca-1a165ab8f55b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895177138Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895188856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711,"session_id":"8d54f235-e599-4c4c-bfca-1a165ab8f55b"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:13.895200844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 27 len 32768 data: 10 11 11 9 10 10 10 6 02/30 Read block 184 len 40960 03/30 Read block 262 len 20480 04/30 Read block 470 len 12288 05/30 Read block 311 len 32768 06/30 Read block 277 len 40960 07/30 Flush 08/30 Write block 534 len 12288 data: 12 11 7 09/30 Read block 23 len 4096 10/30 Write block 113 len 36864 data: 20 16 15 14 10 12 11 11 10 11/30 Write block 53 len 4096 data: 11 12/30 Write block 41 len 16384 data: 8 11 13 17 13/30 Read block 560 len 8192 14/30 Read block 316 len 36864 15/30 Write block 598 len 4096 data: 4 16/30 Write block 108 len 28672 data: 9 12 11 14 18 21 17 17/30 Read block 585 len 32768 18/30 Write block 131 len 36864 data: 9 9 9 8 9 9 9 8 7 19/30 Write block 517 len 20480 data: 10 11 9 10 10 20/30 Read block 229 len 16384 21/30 Read block 55 len 20480 22/30 Read block 346 len 36864 23/30 Write block 34 len 28672 data: 7 5 5 5 4 4 6 24/30 Write block 429 len 36864 data: 12 12 13 14 13 13 10 12 13 25/30 Write block 547 len 16384 data: 7 7 6 8 26/30 Write block 265 len 16384 data: 10 6 7 7 27/30 Write block 363 len 4096 data: 10 28/30 Read block 490 len 24576 29/30 Read block 404 len 8192 30/30 Read block 445 len 20480 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 4 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false 31 Acked 1030 Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 31 30 29 24 28 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:13.919800778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1711} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 132 132 132 266 266 266 F F F 2 040-059 132 132 132 266 266 266 F F F 3 060-079 128 128 128 256 256 256 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 132 132 132 266 266 266 F F F 6 120-139 132 132 132 266 266 266 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 129 129 129 257 257 257 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 120 120 120 240 240 240 T T T 14 280-299 123 123 123 246 246 246 F F F 15 300-319 131 131 131 264 264 264 F F F 16 320-339 131 131 131 264 264 264 F F F 17 340-359 128 128 128 255 255 255 F F F 18 360-379 130 130 130 258 258 258 T T T 19 380-399 131 131 131 264 264 264 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 126 126 126 252 252 252 T T T 22 440-459 124 124 124 250 250 250 F F F 23 460-479 130 130 130 258 258 258 F F F 24 480-499 130 130 130 260 260 260 F F F 25 500-519 132 132 132 266 266 266 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 129 129 129 257 257 257 T T T 28 560-579 124 124 124 248 248 248 F F F 29 580-599 132 132 132 266 266 266 F F F Max gen: 132, Max flush: 266 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 132 132 132 266 266 266 F F F 2 040-059 132 132 132 266 266 266 F F F 3 060-079 128 128 128 256 256 256 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 132 132 132 266 266 266 F F F 6 120-139 132 132 132 266 266 266 F F F 7 140-159 123 123 123 246 246 246 F F F 8 160-179 129 129 129 257 257 257 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 133 133 133 267 267 267 F F F 14 280-299 123 123 123 246 246 246 F F F 15 300-319 131 131 131 264 264 264 F F F 16 320-339 131 131 131 264 264 264 F F F 17 340-359 128 128 128 255 255 255 F F F 18 360-379 133 133 133 267 267 267 F F F 19 380-399 131 131 131 264 264 264 F F F 20 400-419 126 126 126 252 252 252 F F F 21 420-439 133 133 133 267 267 267 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 130 130 130 258 258 258 F F F 24 480-499 130 130 130 260 260 260 F F F 25 500-519 132 132 132 266 266 266 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 133 133 133 267 267 267 F F F 28 560-579 124 124 124 248 248 248 F F F 29 580-599 132 132 132 266 266 266 F F F Max gen: 133, Max flush: 267 Begin loop 67 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.126162536Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.126583593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.126616458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719} {"msg":"Crucible aedccca1-53e9-4973-9bb6-146c3d3c7c17 has session id: 3be5a81d-41d3-4ac2-97ee-c7330be15e52","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.126668231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Upstairs opts: Upstairs UUID: aedccca1-53e9-4973-9bb6-146c3d3c7c17, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.126784433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Crucible stats registered with UUID: aedccca1-53e9-4973-9bb6-146c3d3c7c17","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.126812289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127238196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719} {"msg":"aedccca1-53e9-4973-9bb6-146c3d3c7c17 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127261913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.12727934Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127305436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127331063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127354429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127378236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127401972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127423919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127451495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127476781Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:28.127594673Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:56783)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127630288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:28.127659384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:28.12768817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127712616Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127787015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:36576)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.12781794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.127842417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.128342622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.128368529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.128383427Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.128397385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.12890119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.129015393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.129353303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.129389967Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.129403905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.129550634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.12957682Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:28.129591388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.12909749Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_connection connected from Ok(127.0.0.1:54021)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.129275354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"io task","client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.130231802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.130259588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.130848081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131422856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131450682Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 266, 256, 255, 266, 266, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131467419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[0]R generation[0..12]: [127, 132, 132, 128, 128, 132, 132, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131486616Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131501064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 266, 256, 255, 266, 266, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131514832Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[1]R generation[0..12]: [127, 132, 132, 128, 128, 132, 132, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.13152854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131542558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 266, 256, 255, 266, 266, 246, 257, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131556176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[2]R generation[0..12]: [127, 132, 132, 128, 128, 132, 132, 123, 129, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131569734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-08-05T19:35:38.131583612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.13160027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131613058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Next flush: 268","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131625726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131638774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"":"downstairs","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131651802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.13166368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"0","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131679048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"1","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131692116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"client":"2","":"downstairs","session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131705324Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131717272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"aedccca1-53e9-4973-9bb6-146c3d3c7c17 is now active with session: 3be5a81d-41d3-4ac2-97ee-c7330be15e52","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.13172983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131741519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719,"session_id":"3be5a81d-41d3-4ac2-97ee-c7330be15e52"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:38.131754377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 40960 data: 9 9 10 9 10 9 9 9 9 12 02/30 Write block 353 len 32768 data: 10 11 11 7 5 6 9 10 03/30 Read block 463 len 12288 04/30 Write block 500 len 24576 data: 14 14 12 14 13 13 05/30 Read block 352 len 24576 06/30 Read block 465 len 16384 07/30 Read block 514 len 8192 08/30 Write block 395 len 24576 data: 12 16 14 12 11 14 09/30 Write block 494 len 40960 data: 11 12 15 18 16 14 15 15 13 15 10/30 Read block 134 len 8192 11/30 Write block 366 len 20480 data: 14 17 16 14 13 12/30 Write block 150 len 24576 data: 8 8 6 8 8 8 13/30 Read block 0 len 36864 14/30 Read block 174 len 4096 15/30 Write block 289 len 8192 data: 13 14 16/30 Flush 17/30 Write block 389 len 28672 data: 14 14 13 12 11 12 13 18/30 Write block 170 len 32768 data: 14 13 11 11 10 11 7 8 19/30 Write block 559 len 40960 data: 8 9 10 10 13 11 10 11 11 14 20/30 Write block 566 len 32768 data: 12 12 15 12 13 17 15 14 21/30 Write block 584 len 32768 data: 7 9 10 8 8 8 11 9 22/30 Read block 486 len 12288 23/30 Write block 68 len 4096 data: 9 24/30 Read block 470 len 28672 25/30 Read block 346 len 36864 26/30 Flush 27/30 Write block 286 len 28672 data: 9 12 12 14 15 15 13 28/30 Read block 123 len 20480 29/30 Read block 449 len 36864 30/30 Write block 478 len 28672 data: 13 13 14 13 17 17 19 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 5 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:38.160343799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1719} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 132 132 132 266 266 266 F F F 2 040-059 132 132 132 266 266 266 F F F 3 060-079 134 134 134 269 269 269 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 132 132 132 266 266 266 F F F 6 120-139 132 132 132 266 266 266 F F F 7 140-159 134 134 134 268 268 268 F F F 8 160-179 134 134 134 269 269 269 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 133 133 133 267 267 267 F F F 14 280-299 134 134 134 268 268 268 T T T 15 300-319 131 131 131 264 264 264 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 134 134 134 268 268 268 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 134 134 134 269 269 269 F F F 20 400-419 134 134 134 268 268 268 F F F 21 420-439 133 133 133 267 267 267 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 130 130 130 258 258 258 T T T 24 480-499 134 134 134 268 268 268 T T T 25 500-519 134 134 134 268 268 268 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 134 134 134 269 269 269 F F F Max gen: 134, Max flush: 269 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 F F F 1 020-039 132 132 132 266 266 266 F F F 2 040-059 132 132 132 266 266 266 F F F 3 060-079 134 134 134 269 269 269 F F F 4 080-099 128 128 128 255 255 255 F F F 5 100-119 132 132 132 266 266 266 F F F 6 120-139 132 132 132 266 266 266 F F F 7 140-159 134 134 134 268 268 268 F F F 8 160-179 134 134 134 269 269 269 F F F 9 180-199 128 128 128 256 256 256 F F F 10 200-219 129 129 129 257 257 257 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 133 133 133 267 267 267 F F F 14 280-299 135 135 135 270 270 270 F F F 15 300-319 131 131 131 264 264 264 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 134 134 134 268 268 268 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 134 134 134 269 269 269 F F F 20 400-419 134 134 134 268 268 268 F F F 21 420-439 133 133 133 267 267 267 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 135 135 135 270 270 270 F F F 25 500-519 134 134 134 268 268 268 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 134 134 134 269 269 269 F F F Max gen: 135, Max flush: 270 Begin loop 68 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.359701494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.360344549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.360376834Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727} {"msg":"Crucible ac9a1e95-252d-44b0-a913-7b74a3652890 has session id: aa4bbc98-5725-43f8-baa0-d29495581edb","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.360528852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Upstairs opts: Upstairs UUID: ac9a1e95-252d-44b0-a913-7b74a3652890, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.360631186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Crucible stats registered with UUID: ac9a1e95-252d-44b0-a913-7b74a3652890","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.360660732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.360918634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727} {"msg":"ac9a1e95-252d-44b0-a913-7b74a3652890 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361032987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361051484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361066282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.36107981Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361093658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361107116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361130183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361151289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361170457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361184335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:35409)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361315625Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56725)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.36135319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:52.361643636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:52.361673462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:35:52.361691689Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361713356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361737502Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.361764709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.362188156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.362222441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.362249257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.362276982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.362753782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.362885312Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.363354693Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.363378809Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.363392817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.363404835Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.363418084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:35:52.363431111Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.362250385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_connection connected from Ok(127.0.0.1:42277)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.362318735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"io task","client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.363515038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.363557991Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364163962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364737727Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364765532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[0]R flush_numbers[0..12]: [254, 266, 266, 269, 255, 266, 266, 268, 269, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.36478219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[0]R generation[0..12]: [127, 132, 132, 134, 128, 132, 132, 134, 134, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364803137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364817885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[1]R flush_numbers[0..12]: [254, 266, 266, 269, 255, 266, 266, 268, 269, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364831783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[1]R generation[0..12]: [127, 132, 132, 134, 128, 132, 132, 134, 134, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364845641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364859629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[2]R flush_numbers[0..12]: [254, 266, 266, 269, 255, 266, 266, 268, 269, 256, 257, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364873436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[2]R generation[0..12]: [127, 132, 132, 134, 128, 132, 132, 134, 134, 128, 129, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364887345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364901262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364914141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Generation requested: 136 >= found:136","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2024-08-05T19:36:02.364944646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364958014Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364970942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"":"downstairs","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.36498439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.364996808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"0","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.365012526Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"1","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.365025694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"client":"2","":"downstairs","session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.365039072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.36505112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"ac9a1e95-252d-44b0-a913-7b74a3652890 is now active with session: aa4bbc98-5725-43f8-baa0-d29495581edb","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.365063888Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.365075686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727,"session_id":"aa4bbc98-5725-43f8-baa0-d29495581edb"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:02.365087675Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 205 len 36864 data: 14 15 18 15 13 13 15 13 10 03/30 Read block 353 len 16384 04/30 Read block 478 len 40960 05/30 Read block 522 len 8192 06/30 Read block 215 len 28672 07/30 Read block 28 len 32768 08/30 Write block 492 len 12288 data: 9 8 12 09/30 Read block 384 len 20480 10/30 Read block 402 len 24576 11/30 Write block 202 len 40960 data: 10 9 10 15 16 19 16 14 14 16 12/30 Flush 13/30 Write block 96 len 24576 data: 9 8 9 8 7 8 14/30 Read block 423 len 20480 15/30 Write block 135 len 24576 data: 10 10 10 9 8 7 16/30 Read block 108 len 24576 17/30 Write block 94 len 24576 data: 8 10 10 9 10 9 18/30 Read block 174 len 40960 19/30 Write block 69 len 24576 data: 12 6 4 5 5 5 20/30 Read block 541 len 8192 21/30 Write block 430 len 12288 data: 13 14 15 22/30 Read block 214 len 8192 23/30 Read block 23 len 12288 24/30 Read block 358 len 36864 25/30 Read block 531 len 40960 26/30 Write block 191 len 20480 data: 8 8 7 6 9 27/30 Write block 4 len 28672 data: 7 9 9 9 9 12 13 28/30 Write block 27 len 20480 data: 11 12 12 10 11 29/30 Read block 398 len 32768 30/30 Read block 219 len 24576 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 3 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Write 5 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Read 6 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 10 9 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:02.39237642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1727} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 127 127 127 254 254 254 T T T 1 020-039 132 132 132 266 266 266 T T T 2 040-059 132 132 132 266 266 266 F F F 3 060-079 136 136 136 273 273 273 F F F 4 080-099 136 136 136 273 273 273 F F F 5 100-119 136 136 136 273 273 273 F F F 6 120-139 136 136 136 273 273 273 F F F 7 140-159 136 136 136 273 273 273 F F F 8 160-179 134 134 134 269 269 269 F F F 9 180-199 128 128 128 256 256 256 T T T 10 200-219 136 136 136 272 272 272 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 133 133 133 267 267 267 F F F 14 280-299 135 135 135 270 270 270 F F F 15 300-319 131 131 131 264 264 264 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 134 134 134 268 268 268 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 134 134 134 269 269 269 F F F 20 400-419 134 134 134 268 268 268 F F F 21 420-439 133 133 133 267 267 267 T T T 22 440-459 124 124 124 250 250 250 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 136 136 136 272 272 272 F F F 25 500-519 134 134 134 268 268 268 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 134 134 134 269 269 269 F F F Max gen: 136, Max flush: 273 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 132 132 132 266 266 266 F F F 3 060-079 136 136 136 273 273 273 F F F 4 080-099 136 136 136 273 273 273 F F F 5 100-119 136 136 136 273 273 273 F F F 6 120-139 136 136 136 273 273 273 F F F 7 140-159 136 136 136 273 273 273 F F F 8 160-179 134 134 134 269 269 269 F F F 9 180-199 137 137 137 274 274 274 F F F 10 200-219 136 136 136 272 272 272 F F F 11 220-239 122 122 122 244 244 244 F F F 12 240-259 122 122 122 243 243 243 F F F 13 260-279 133 133 133 267 267 267 F F F 14 280-299 135 135 135 270 270 270 F F F 15 300-319 131 131 131 264 264 264 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 134 134 134 268 268 268 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 134 134 134 269 269 269 F F F 20 400-419 134 134 134 268 268 268 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 136 136 136 272 272 272 F F F 25 500-519 134 134 134 268 268 268 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 134 134 134 269 269 269 F F F Max gen: 137, Max flush: 274 Begin loop 69 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.603959394Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.604371553Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.60439444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735} {"msg":"Crucible 8870f60a-10ab-4817-af04-a82a8f5e8f9d has session id: 3a7eadf4-894a-4478-9818-251139b3f5cc","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.604680347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Upstairs opts: Upstairs UUID: 8870f60a-10ab-4817-af04-a82a8f5e8f9d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.604823876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Crucible stats registered with UUID: 8870f60a-10ab-4817-af04-a82a8f5e8f9d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.604849192Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.60513696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735} {"msg":"8870f60a-10ab-4817-af04-a82a8f5e8f9d active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605244064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605263381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605278539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605300046Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605326291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605352398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605376914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.60540617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605432336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605458462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35506)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605489887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:16.605520753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:16.605549338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:16.605576374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605596722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605620348Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:44437)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605645384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.605671311Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.60635263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.606385765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.606412801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.606478361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607003293Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607033338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607570399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607597185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607612143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.60762721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607640939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:16.607654077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.606486499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:50897)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.606609081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"io task","client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.607656026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.607693021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.60823931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.60884095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608861087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[0]R flush_numbers[0..12]: [274, 274, 266, 273, 273, 273, 273, 273, 269, 274, 272, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608880165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[0]R generation[0..12]: [137, 137, 132, 136, 136, 136, 136, 136, 134, 137, 136, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608894782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.60890903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[1]R flush_numbers[0..12]: [274, 274, 266, 273, 273, 273, 273, 273, 269, 274, 272, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608922908Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[1]R generation[0..12]: [137, 137, 132, 136, 136, 136, 136, 136, 134, 137, 136, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608936626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608950894Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[2]R flush_numbers[0..12]: [274, 274, 266, 273, 273, 273, 273, 273, 269, 274, 272, 244]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608964812Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[2]R generation[0..12]: [137, 137, 132, 136, 136, 136, 136, 136, 134, 137, 136, 122]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.60897854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.608992518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609019464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609032912Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609049499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609063497Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"":"downstairs","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609078455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609093003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"0","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609106501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"1","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609119829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"client":"2","":"downstairs","session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609133187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609145235Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"8870f60a-10ab-4817-af04-a82a8f5e8f9d is now active with session: 3a7eadf4-894a-4478-9818-251139b3f5cc","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609157914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.609169872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735,"session_id":"3a7eadf4-894a-4478-9818-251139b3f5cc"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:26.6091819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 484 len 24576 data: 20 18 17 15 17 17 02/30 Write block 220 len 12288 data: 12 11 8 03/30 Read block 471 len 8192 04/30 Write block 209 len 20480 data: 15 15 17 14 11 05/30 Write block 291 len 40960 data: 16 14 10 11 9 9 10 9 11 11 06/30 Read block 113 len 4096 07/30 Flush 08/30 Write block 77 len 36864 data: 9 11 12 11 10 12 13 12 8 09/30 Read block 497 len 36864 10/30 Read block 259 len 40960 11/30 Write block 414 len 8192 data: 9 8 12/30 Read block 446 len 32768 13/30 Read block 310 len 12288 14/30 Write block 295 len 24576 data: 10 10 11 10 12 12 15/30 Read block 75 len 40960 16/30 Write block 475 len 36864 data: 9 12 11 14 14 15 14 18 18 17/30 Write block 283 len 36864 data: 8 8 7 10 13 13 15 16 17 18/30 Write block 481 len 4096 data: 15 19/30 Write block 48 len 28672 data: 10 10 8 9 10 12 10 20/30 Write block 285 len 4096 data: 8 21/30 Write block 96 len 32768 data: 11 10 11 10 8 9 8 8 22/30 Read block 170 len 12288 23/30 Write block 253 len 20480 data: 15 14 9 11 12 24/30 Write block 242 len 4096 data: 8 25/30 Read block 437 len 36864 26/30 Write block 380 len 28672 data: 18 15 12 9 9 8 9 27/30 Read block 74 len 36864 28/30 Read block 139 len 12288 29/30 Read block 94 len 24576 30/30 Write block 55 len 12288 data: 13 11 12 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 3 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Write 1 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:26.635505368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1735} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 138 138 138 276 276 276 T T T 3 060-079 138 138 138 276 276 276 F F F 4 080-099 138 138 138 276 276 276 T T T 5 100-119 136 136 136 273 273 273 T T T 6 120-139 136 136 136 273 273 273 F F F 7 140-159 136 136 136 273 273 273 F F F 8 160-179 134 134 134 269 269 269 F F F 9 180-199 137 137 137 274 274 274 F F F 10 200-219 138 138 138 275 275 275 F F F 11 220-239 138 138 138 275 275 275 F F F 12 240-259 122 122 122 243 243 243 T T T 13 260-279 133 133 133 267 267 267 F F F 14 280-299 138 138 138 276 276 276 F F F 15 300-319 138 138 138 276 276 276 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 134 134 134 268 268 268 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 134 134 134 269 269 269 T T T 20 400-419 138 138 138 276 276 276 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 134 134 134 268 268 268 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 134 134 134 269 269 269 F F F Max gen: 138, Max flush: 276 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 139 139 139 277 277 277 F F F 3 060-079 138 138 138 276 276 276 F F F 4 080-099 139 139 139 277 277 277 F F F 5 100-119 139 139 139 277 277 277 F F F 6 120-139 136 136 136 273 273 273 F F F 7 140-159 136 136 136 273 273 273 F F F 8 160-179 134 134 134 269 269 269 F F F 9 180-199 137 137 137 274 274 274 F F F 10 200-219 138 138 138 275 275 275 F F F 11 220-239 138 138 138 275 275 275 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 133 133 133 267 267 267 F F F 14 280-299 138 138 138 276 276 276 F F F 15 300-319 138 138 138 276 276 276 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 134 134 134 268 268 268 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 139 139 139 277 277 277 F F F 20 400-419 138 138 138 276 276 276 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 134 134 134 268 268 268 F F F 26 520-539 132 132 132 266 266 266 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 134 134 134 269 269 269 F F F Max gen: 139, Max flush: 277 Begin loop 70 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.860498534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.860972384Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.86100072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743} {"msg":"Crucible 84a7a5cc-bc7d-4daf-a3ee-17b5a6f77057 has session id: a04145b3-60f6-4d09-8c9b-6ab6c3787eed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861147358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Upstairs opts: Upstairs UUID: 84a7a5cc-bc7d-4daf-a3ee-17b5a6f77057, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861190161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Crucible stats registered with UUID: 84a7a5cc-bc7d-4daf-a3ee-17b5a6f77057","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861214138Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861511424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743} {"msg":"84a7a5cc-bc7d-4daf-a3ee-17b5a6f77057 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861666361Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861684268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861703745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861727792Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861753688Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861780574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.86180597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861828167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.861852253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.8618766Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:40.862033826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:40.86207465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_connection connected from Ok(127.0.0.1:51130)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862102576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:40.862131482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862237706Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862262053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862284789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_connection connected from Ok(127.0.0.1:35534)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862320024Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862801702Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862839247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862871772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.862898498Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.86342239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.863532904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.863985307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.864012983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.864028561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.864040909Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.864054477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:40.864067655Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.863857856Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_connection connected from Ok(127.0.0.1:50434)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.864055537Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"io task","client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.865008215Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.86504746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.865630603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866233424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866269299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[0]R flush_numbers[0..12]: [274, 274, 277, 276, 277, 277, 273, 273, 269, 274, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866286016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[0]R generation[0..12]: [137, 137, 139, 138, 139, 139, 136, 136, 134, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866300744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866315242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[1]R flush_numbers[0..12]: [274, 274, 277, 276, 277, 277, 273, 273, 269, 274, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.86632902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[1]R generation[0..12]: [137, 137, 139, 138, 139, 139, 136, 136, 134, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866342868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866356986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[2]R flush_numbers[0..12]: [274, 274, 277, 276, 277, 277, 273, 273, 269, 274, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866370893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[2]R generation[0..12]: [137, 137, 139, 138, 139, 139, 136, 136, 134, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866384491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:36:50.866398389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866411307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866424405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866437124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866453621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"":"downstairs","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866466799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866478608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"0","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866491726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"1","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866504534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"client":"2","":"downstairs","session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866517452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.86652951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"84a7a5cc-bc7d-4daf-a3ee-17b5a6f77057 is now active with session: a04145b3-60f6-4d09-8c9b-6ab6c3787eed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866542288Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866554046Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743,"session_id":"a04145b3-60f6-4d09-8c9b-6ab6c3787eed"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:36:50.866566104Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 409 len 8192 data: 13 12 02/30 Read block 195 len 8192 03/30 Flush 04/30 Write block 265 len 12288 data: 11 7 8 05/30 Read block 529 len 36864 06/30 Read block 478 len 4096 07/30 Write block 351 len 24576 data: 10 11 11 12 12 8 08/30 Write block 159 len 20480 data: 14 16 14 17 16 09/30 Write block 123 len 8192 data: 11 8 10/30 Read block 116 len 4096 11/30 Write block 582 len 8192 data: 10 7 12/30 Read block 478 len 40960 13/30 Read block 242 len 16384 14/30 Flush 15/30 Write block 299 len 36864 data: 13 13 11 11 11 11 10 9 9 16/30 Read block 8 len 40960 17/30 Write block 76 len 24576 data: 7 10 12 13 12 11 18/30 Read block 587 len 24576 19/30 Write block 285 len 12288 data: 9 11 14 20/30 Read block 567 len 8192 21/30 Write block 520 len 40960 data: 11 11 9 10 10 9 8 9 8 7 22/30 Write block 137 len 40960 data: 11 10 9 8 8 10 11 12 14 12 23/30 Write block 345 len 20480 data: 13 13 9 10 10 24/30 Read block 458 len 32768 25/30 Read block 125 len 8192 26/30 Read block 362 len 32768 27/30 Read block 571 len 32768 28/30 Write block 66 len 32768 data: 12 11 10 13 7 5 6 6 29/30 Write block 511 len 28672 data: 15 15 12 15 15 13 11 30/30 Write block 81 len 32768 data: 12 13 14 13 9 7 6 8 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Write 6 Done Done Done false 19 Acked 1018 Read 6 Done Done Done false 20 Acked 1019 Write 3 Done Done Done false 21 Acked 1020 Read 2 Done Done Done false 22 Acked 1021 Write 10 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Write 7 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:36:50.892257007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1743} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 139 139 139 277 277 277 F F F 3 060-079 138 138 138 276 276 276 T T T 4 080-099 139 139 139 277 277 277 T T T 5 100-119 139 139 139 277 277 277 F F F 6 120-139 140 140 140 279 279 279 T T T 7 140-159 140 140 140 279 279 279 T T T 8 160-179 140 140 140 279 279 279 F F F 9 180-199 137 137 137 274 274 274 F F F 10 200-219 138 138 138 275 275 275 F F F 11 220-239 138 138 138 275 275 275 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 140 140 140 279 279 279 F F F 14 280-299 140 140 140 280 280 280 T T T 15 300-319 140 140 140 280 280 280 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 140 140 140 279 279 279 T T T 18 360-379 134 134 134 268 268 268 F F F 19 380-399 139 139 139 277 277 277 F F F 20 400-419 140 140 140 278 278 278 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 134 134 134 268 268 268 T T T 26 520-539 132 132 132 266 266 266 T T T 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 140 140 140 279 279 279 F F F Max gen: 140, Max flush: 280 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 139 139 139 277 277 277 F F F 3 060-079 141 141 141 281 281 281 F F F 4 080-099 141 141 141 281 281 281 F F F 5 100-119 139 139 139 277 277 277 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 141 141 141 281 281 281 F F F 8 160-179 140 140 140 279 279 279 F F F 9 180-199 137 137 137 274 274 274 F F F 10 200-219 138 138 138 275 275 275 F F F 11 220-239 138 138 138 275 275 275 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 140 140 140 279 279 279 F F F 14 280-299 141 141 141 281 281 281 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 134 134 134 268 268 268 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 139 139 139 277 277 277 F F F 20 400-419 140 140 140 278 278 278 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 138 138 138 276 276 276 F F F 25 500-519 141 141 141 281 281 281 F F F 26 520-539 141 141 141 281 281 281 F F F 27 540-559 134 134 134 269 269 269 F F F 28 560-579 134 134 134 269 269 269 F F F 29 580-599 140 140 140 279 279 279 F F F Max gen: 141, Max flush: 281 Begin loop 71 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.155153695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.155759525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.15579156Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751} {"msg":"Crucible 651f48c7-75dd-4e9a-babf-0a8bbfec571b has session id: ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.155899484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Upstairs opts: Upstairs UUID: 651f48c7-75dd-4e9a-babf-0a8bbfec571b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.155940398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Crucible stats registered with UUID: 651f48c7-75dd-4e9a-babf-0a8bbfec571b","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.155966414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156257011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751} {"msg":"651f48c7-75dd-4e9a-babf-0a8bbfec571b active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156364385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156383722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156403829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156417997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156431365Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156452262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156476419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156491066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156505174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156519012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57210)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156697036Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38290)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.15673387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:05.156904835Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:05.156946199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:05.156976274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.156990842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.157012779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.157119543Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.157627668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.157654154Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.157668882Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.1576825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.158379406Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.158407262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.159010363Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.159038609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.159052866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.159065895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.159079213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:05.159092311Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.157450174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62943)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.157553759Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"io task","client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.158687551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.158720766Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.159395406Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.159990238Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160018064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[0]R flush_numbers[0..12]: [274, 274, 277, 281, 281, 277, 281, 281, 279, 274, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160034661Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[0]R generation[0..12]: [137, 137, 139, 141, 141, 139, 141, 141, 140, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160049599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160063697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[1]R flush_numbers[0..12]: [274, 274, 277, 281, 281, 277, 281, 281, 279, 274, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160077465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[1]R generation[0..12]: [137, 137, 139, 141, 141, 139, 141, 141, 140, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160091103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160105121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[2]R flush_numbers[0..12]: [274, 274, 277, 281, 281, 277, 281, 281, 279, 274, 275, 275]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160118569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[2]R generation[0..12]: [137, 137, 139, 141, 141, 139, 141, 141, 140, 137, 138, 138]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160132077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160145985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160162592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.16017966Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160193388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160206216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"":"downstairs","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160219154Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160231132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"0","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.16024413Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"1","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160257008Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"client":"2","":"downstairs","session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160269986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160282724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"651f48c7-75dd-4e9a-babf-0a8bbfec571b is now active with session: ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160295713Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160307441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751,"session_id":"ed5d9fe2-ab39-457e-91a4-a49ee45bf3c5"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:15.160319299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 570 len 28672 data: 14 18 16 15 14 14 12 02/30 Flush 03/30 Write block 322 len 28672 data: 6 8 10 10 11 10 11 04/30 Write block 571 len 32768 data: 19 17 16 15 15 13 10 11 05/30 Read block 584 len 36864 06/30 Read block 410 len 4096 07/30 Read block 1 len 4096 08/30 Write block 109 len 12288 data: 13 12 15 09/30 Write block 165 len 24576 data: 16 15 16 14 16 15 10/30 Write block 66 len 24576 data: 13 12 11 14 8 6 11/30 Read block 548 len 36864 12/30 Write block 232 len 8192 data: 8 7 13/30 Read block 516 len 16384 14/30 Read block 175 len 24576 15/30 Write block 165 len 32768 data: 17 16 17 15 17 16 14 12 16/30 Read block 39 len 28672 17/30 Write block 514 len 24576 data: 16 16 14 12 12 10 18/30 Write block 222 len 16384 data: 9 7 4 5 19/30 Write block 149 len 28672 data: 9 9 9 7 9 9 9 20/30 Read block 219 len 24576 21/30 Read block 53 len 20480 22/30 Flush 23/30 Write block 397 len 4096 data: 15 24/30 Write block 192 len 36864 data: 9 8 7 10 7 7 5 8 9 25/30 Read block 24 len 24576 26/30 Read block 311 len 40960 27/30 Write block 546 len 20480 data: 7 8 8 7 9 28/30 Write block 484 len 12288 data: 21 19 18 29/30 Read block 312 len 8192 30/30 Read block 57 len 16384 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 6 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false 31 Acked 1030 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:15.187495381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1751} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 139 139 139 277 277 277 F F F 3 060-079 142 142 142 283 283 283 F F F 4 080-099 141 141 141 281 281 281 F F F 5 100-119 142 142 142 283 283 283 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 142 142 142 283 283 283 F F F 8 160-179 142 142 142 283 283 283 F F F 9 180-199 137 137 137 274 274 274 T T T 10 200-219 138 138 138 275 275 275 T T T 11 220-239 142 142 142 283 283 283 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 140 140 140 279 279 279 F F F 14 280-299 141 141 141 281 281 281 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 142 142 142 283 283 283 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 139 139 139 277 277 277 T T T 20 400-419 140 140 140 278 278 278 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 138 138 138 276 276 276 T T T 25 500-519 142 142 142 283 283 283 F F F 26 520-539 141 141 141 281 281 281 F F F 27 540-559 134 134 134 269 269 269 T T T 28 560-579 142 142 142 283 283 283 F F F 29 580-599 140 140 140 279 279 279 F F F Max gen: 142, Max flush: 283 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 F F F 2 040-059 139 139 139 277 277 277 F F F 3 060-079 142 142 142 283 283 283 F F F 4 080-099 141 141 141 281 281 281 F F F 5 100-119 142 142 142 283 283 283 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 142 142 142 283 283 283 F F F 8 160-179 142 142 142 283 283 283 F F F 9 180-199 143 143 143 284 284 284 F F F 10 200-219 143 143 143 284 284 284 F F F 11 220-239 142 142 142 283 283 283 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 140 140 140 279 279 279 F F F 14 280-299 141 141 141 281 281 281 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 142 142 142 283 283 283 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 134 134 134 268 268 268 F F F 19 380-399 143 143 143 284 284 284 F F F 20 400-419 140 140 140 278 278 278 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 143 143 143 284 284 284 F F F 25 500-519 142 142 142 283 283 283 F F F 26 520-539 141 141 141 281 281 281 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 142 142 142 283 283 283 F F F 29 580-599 140 140 140 279 279 279 F F F Max gen: 143, Max flush: 284 Begin loop 72 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.418282149Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.418770357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.418797683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759} {"msg":"Crucible 2f67f778-1053-458c-ad4c-2058cc866271 has session id: 72bcb9d4-17ca-4203-bc7b-6ab892fa723f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.418967647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Upstairs opts: Upstairs UUID: 2f67f778-1053-458c-ad4c-2058cc866271, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419046486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Crucible stats registered with UUID: 2f67f778-1053-458c-ad4c-2058cc866271","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419072862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419400723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759} {"msg":"2f67f778-1053-458c-ad4c-2058cc866271 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419425729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419449086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419475602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419501488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419524225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419548311Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419572648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419595994Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.41962138Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419645387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:29.419738203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:29.419772238Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57799)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419796574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:29.41982391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419846187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419868944Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_connection connected from Ok(127.0.0.1:42727)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.41989785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.419921806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.420487372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.420520137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.420546813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.420610094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421199067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421236521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421833652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421855399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421869287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421881305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421894614Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:29.421907642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.421535137Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33867)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.421705642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"io task","client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.422649972Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.422677848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423286857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423898467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423917494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[0]R flush_numbers[0..12]: [274, 274, 277, 283, 281, 283, 281, 283, 283, 284, 284, 283]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423931732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[0]R generation[0..12]: [137, 137, 139, 142, 141, 142, 141, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.42394575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423959977Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[1]R flush_numbers[0..12]: [274, 274, 277, 283, 281, 283, 281, 283, 283, 284, 284, 283]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423973995Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[1]R generation[0..12]: [137, 137, 139, 142, 141, 142, 141, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.423987753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424024398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[2]R flush_numbers[0..12]: [274, 274, 277, 283, 281, 283, 281, 283, 283, 284, 284, 283]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424047545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[2]R generation[0..12]: [137, 137, 139, 142, 141, 142, 141, 142, 142, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424067792Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424086219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424103576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424117104Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424129952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.42414285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"":"downstairs","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424155708Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424167427Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"0","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424180765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"1","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424193923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"client":"2","":"downstairs","session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424206951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424218979Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"2f67f778-1053-458c-ad4c-2058cc866271 is now active with session: 72bcb9d4-17ca-4203-bc7b-6ab892fa723f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424231577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424243346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759,"session_id":"72bcb9d4-17ca-4203-bc7b-6ab892fa723f"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:39.424255354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 284 len 4096 data: 9 02/30 Write block 260 len 24576 data: 9 10 9 8 8 12 03/30 Write block 50 len 12288 data: 9 10 11 04/30 Read block 504 len 8192 05/30 Flush 06/30 Write block 93 len 32768 data: 10 9 11 12 11 12 11 9 07/30 Write block 490 len 40960 data: 14 11 10 9 13 13 16 19 17 15 08/30 Write block 174 len 24576 data: 11 12 8 9 7 7 09/30 Write block 504 len 28672 data: 14 14 13 12 14 17 16 10/30 Write block 488 len 28672 data: 18 18 15 12 11 10 14 11/30 Read block 537 len 32768 12/30 Read block 574 len 36864 13/30 Read block 251 len 8192 14/30 Read block 261 len 36864 15/30 Read block 23 len 40960 16/30 Read block 274 len 32768 17/30 Read block 494 len 32768 18/30 Write block 515 len 36864 data: 17 15 13 13 11 12 12 10 11 19/30 Flush 20/30 Write block 393 len 20480 data: 12 13 14 17 16 21/30 Flush 22/30 Write block 162 len 40960 data: 18 17 16 18 17 18 16 18 17 15 23/30 Write block 501 len 4096 data: 16 24/30 Write block 332 len 8192 data: 10 13 25/30 Read block 372 len 20480 26/30 Read block 99 len 32768 27/30 Read block 218 len 40960 28/30 Write block 35 len 8192 data: 6 6 29/30 Write block 367 len 16384 data: 18 17 15 14 30/30 Read block 54 len 20480 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Write 1 Done Done Done false 25 Acked 1024 Write 2 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 Acked 1026 Read 8 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Write 4 Done Done Done false 31 Acked 1030 Read 5 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 0 1 Done 9 8 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:39.453643978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1759} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 137 137 137 274 274 274 T T T 2 040-059 144 144 144 285 285 285 F F F 3 060-079 142 142 142 283 283 283 F F F 4 080-099 144 144 144 286 286 286 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 142 142 142 283 283 283 F F F 8 160-179 144 144 144 286 286 286 T T T 9 180-199 143 143 143 284 284 284 F F F 10 200-219 143 143 143 284 284 284 F F F 11 220-239 142 142 142 283 283 283 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 144 144 144 285 285 285 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 142 142 142 283 283 283 T T T 17 340-359 141 141 141 281 281 281 F F F 18 360-379 134 134 134 268 268 268 T T T 19 380-399 144 144 144 288 288 288 F F F 20 400-419 140 140 140 278 278 278 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 144 144 144 286 286 286 F F F 25 500-519 144 144 144 287 287 287 T T T 26 520-539 144 144 144 287 287 287 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 142 142 142 283 283 283 F F F 29 580-599 140 140 140 279 279 279 F F F Max gen: 144, Max flush: 288 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 137 137 137 274 274 274 F F F 1 020-039 145 145 145 289 289 289 F F F 2 040-059 144 144 144 285 285 285 F F F 3 060-079 142 142 142 283 283 283 F F F 4 080-099 144 144 144 286 286 286 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 142 142 142 283 283 283 F F F 8 160-179 145 145 145 289 289 289 F F F 9 180-199 143 143 143 284 284 284 F F F 10 200-219 143 143 143 284 284 284 F F F 11 220-239 142 142 142 283 283 283 F F F 12 240-259 139 139 139 277 277 277 F F F 13 260-279 144 144 144 285 285 285 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 145 145 145 289 289 289 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 145 145 145 289 289 289 F F F 19 380-399 144 144 144 288 288 288 F F F 20 400-419 140 140 140 278 278 278 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 144 144 144 286 286 286 F F F 25 500-519 145 145 145 289 289 289 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 142 142 142 283 283 283 F F F 29 580-599 140 140 140 279 279 279 F F F Max gen: 145, Max flush: 289 Begin loop 73 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.681467414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.681881103Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.681907369Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767} {"msg":"Crucible 701e475b-e66a-464e-8fd2-e8e2eb3cef69 has session id: 6d85d365-b0f0-4f32-a718-fa92a777aad9","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682159562Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Upstairs opts: Upstairs UUID: 701e475b-e66a-464e-8fd2-e8e2eb3cef69, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682216573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Crucible stats registered with UUID: 701e475b-e66a-464e-8fd2-e8e2eb3cef69","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.68224161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682527307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767} {"msg":"701e475b-e66a-464e-8fd2-e8e2eb3cef69 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682626023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.68264484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682667797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682693623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682720878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682744375Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682769721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682794448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682818564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.682842051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:53.6829814Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:53.683052639Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_connection connected from Ok(127.0.0.1:41268)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683080925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:37:53.683109811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683132517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683159244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.68318522Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48794)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683213356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683709342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683737678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683752096Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.683773962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.684429855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.684457751Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.684968356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.684996431Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.685011919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.68507411Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.685093397Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:37:53.685106895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.684078237Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_connection connected from Ok(127.0.0.1:49469)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.684143128Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"io task","client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.685190133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.685219068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.685808341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686384326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686412291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[0]R flush_numbers[0..12]: [274, 289, 285, 283, 286, 286, 281, 283, 289, 284, 284, 283]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686429109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[0]R generation[0..12]: [137, 145, 144, 142, 144, 144, 141, 142, 145, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686444257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686458655Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[1]R flush_numbers[0..12]: [274, 289, 285, 283, 286, 286, 281, 283, 289, 284, 284, 283]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686472473Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[1]R generation[0..12]: [137, 145, 144, 142, 144, 144, 141, 142, 145, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686486291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686501588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[2]R flush_numbers[0..12]: [274, 289, 285, 283, 286, 286, 281, 283, 289, 284, 284, 283]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686525305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"Wait for a query_work_queue command to finish before sending IO msg":"[2]R generation[0..12]: [137, 145, 144, 142, 144, 144, 141, 142, 145, 143, 143, 142]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.68655739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686584846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686609452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686630569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Next flush: 290","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686647187Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686664814Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"":"downstairs","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686688171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686711367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"0","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686736643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"1","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.68676149Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"client":"2","":"downstairs","session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686787316Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686803753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"701e475b-e66a-464e-8fd2-e8e2eb3cef69 is now active with session: 6d85d365-b0f0-4f32-a718-fa92a777aad9","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686816692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686833919Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767,"session_id":"6d85d365-b0f0-4f32-a718-fa92a777aad9"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:03.686855726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 582 len 32768 data: 11 8 8 10 11 9 9 9 02/30 Write block 7 len 20480 data: 10 10 13 14 13 03/30 Read block 182 len 28672 04/30 Write block 389 len 24576 data: 15 15 14 13 13 14 05/30 Write block 235 len 36864 data: 5 6 6 8 9 9 8 9 7 06/30 Flush 07/30 Write block 569 len 32768 data: 13 15 20 18 17 16 16 14 08/30 Read block 547 len 20480 09/30 Write block 266 len 4096 data: 8 10/30 Flush 11/30 Write block 148 len 24576 data: 9 10 10 10 8 10 12/30 Read block 581 len 12288 13/30 Write block 210 len 16384 data: 16 18 15 12 14/30 Read block 597 len 12288 15/30 Write block 213 len 28672 data: 13 10 12 12 12 10 12 16/30 Read block 527 len 40960 17/30 Write block 198 len 36864 data: 6 9 10 8 11 10 11 16 17 18/30 Read block 179 len 36864 19/30 Write block 269 len 16384 data: 9 8 10 9 20/30 Read block 468 len 8192 21/30 Write block 493 len 28672 data: 11 15 14 17 20 18 16 22/30 Read block 464 len 36864 23/30 Write block 405 len 24576 data: 17 17 13 11 14 13 24/30 Flush 25/30 Write block 54 len 8192 data: 11 14 26/30 Read block 38 len 12288 27/30 Read block 11 len 40960 28/30 Write block 442 len 8192 data: 11 12 29/30 Read block 27 len 36864 30/30 Read block 452 len 28672 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 10 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Read 7 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:03.716654989Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1767} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 290 290 290 F F F 1 020-039 145 145 145 289 289 289 F F F 2 040-059 144 144 144 285 285 285 T T T 3 060-079 142 142 142 283 283 283 F F F 4 080-099 144 144 144 286 286 286 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 146 146 146 292 292 292 F F F 8 160-179 145 145 145 289 289 289 F F F 9 180-199 146 146 146 293 293 293 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 146 146 146 290 290 290 F F F 12 240-259 146 146 146 290 290 290 F F F 13 260-279 146 146 146 293 293 293 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 145 145 145 289 289 289 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 145 145 145 289 289 289 F F F 19 380-399 146 146 146 290 290 290 F F F 20 400-419 146 146 146 293 293 293 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 124 124 124 250 250 250 T T T 23 460-479 138 138 138 276 276 276 F F F 24 480-499 146 146 146 293 293 293 F F F 25 500-519 145 145 145 289 289 289 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 146 146 146 291 291 291 F F F 29 580-599 146 146 146 290 290 290 F F F Max gen: 146, Max flush: 293 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 146 146 146 290 290 290 F F F 1 020-039 145 145 145 289 289 289 F F F 2 040-059 147 147 147 294 294 294 F F F 3 060-079 142 142 142 283 283 283 F F F 4 080-099 144 144 144 286 286 286 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 146 146 146 292 292 292 F F F 8 160-179 145 145 145 289 289 289 F F F 9 180-199 146 146 146 293 293 293 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 146 146 146 290 290 290 F F F 12 240-259 146 146 146 290 290 290 F F F 13 260-279 146 146 146 293 293 293 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 140 140 140 280 280 280 F F F 16 320-339 145 145 145 289 289 289 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 145 145 145 289 289 289 F F F 19 380-399 146 146 146 290 290 290 F F F 20 400-419 146 146 146 293 293 293 F F F 21 420-439 137 137 137 274 274 274 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 146 146 146 293 293 293 F F F 25 500-519 145 145 145 289 289 289 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 146 146 146 291 291 291 F F F 29 580-599 146 146 146 290 290 290 F F F Max gen: 147, Max flush: 294 Begin loop 74 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.9014663Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.901764695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.901781963Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775} {"msg":"Crucible 9ef82aa6-1e1f-4d08-8b61-23454e596966 has session id: bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.90227067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Upstairs opts: Upstairs UUID: 9ef82aa6-1e1f-4d08-8b61-23454e596966, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902316943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Crucible stats registered with UUID: 9ef82aa6-1e1f-4d08-8b61-23454e596966","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902335461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.9026773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775} {"msg":"9ef82aa6-1e1f-4d08-8b61-23454e596966 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902819299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902842396Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902869132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902898777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902924253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.90294939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902975966Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.902998693Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.903024758Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.903048725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60449)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.903324504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38257)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.903542182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:17.903718776Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:17.903752551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:17.903783936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.903805203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.90382636Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.903847776Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.904115537Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.904142163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.904156591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.904169689Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.904756642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.904783938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.905295852Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.905322568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.905336816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.905349084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.905362182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:17.90537521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.905589188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62117)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.9056481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"io task","client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.906667368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.906695414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907241973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907767405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907795491Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[0]R flush_numbers[0..12]: [290, 289, 294, 283, 286, 286, 281, 292, 289, 293, 293, 290]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907812159Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[0]R generation[0..12]: [146, 145, 147, 142, 144, 144, 141, 146, 145, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907827106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907841424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[1]R flush_numbers[0..12]: [290, 289, 294, 283, 286, 286, 281, 292, 289, 293, 293, 290]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907855212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[1]R generation[0..12]: [146, 145, 147, 142, 144, 144, 141, 146, 145, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.90786892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907883038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[2]R flush_numbers[0..12]: [290, 289, 294, 283, 286, 286, 281, 292, 289, 293, 293, 290]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907896786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"[2]R generation[0..12]: [146, 145, 147, 142, 144, 144, 141, 146, 145, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907915913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":Wait for a query_work_queue command to finish before sending IO "[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907934191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907947199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907960087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907972735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907985373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"":"downstairs","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.907998331Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908010249Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"0","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908023347Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"1","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908036346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"client":"2","":"downstairs","session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908049404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908061392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"9ef82aa6-1e1f-4d08-8b61-23454e596966 is now active with session: bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.90807393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908085658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775,"session_id":"bc29a0d5-6ca4-41a9-b5fb-9af0be2a54e2"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:27.908098196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 367 len 20480 data: 19 18 16 15 12 02/30 Read block 176 len 24576 03/30 Read block 227 len 8192 04/30 Write block 566 len 28672 data: 13 13 16 14 16 21 19 05/30 Flush 06/30 Write block 318 len 36864 data: 8 7 7 6 7 9 11 11 12 07/30 Flush 08/30 Write block 382 len 8192 data: 13 10 09/30 Read block 340 len 4096 10/30 Flush 11/30 Write block 505 len 24576 data: 15 14 13 15 18 17 12/30 Write block 427 len 40960 data: 13 12 13 14 15 16 14 14 11 13 13/30 Read block 299 len 20480 14/30 Read block 486 len 36864 15/30 Flush 16/30 Write block 2 len 24576 data: 5 7 8 10 10 11 17/30 Read block 234 len 16384 18/30 Write block 67 len 24576 data: 13 12 15 9 7 7 19/30 Write block 420 len 8192 data: 8 8 20/30 Read block 315 len 16384 21/30 Write block 168 len 32768 data: 17 19 18 16 13 12 12 13 22/30 Read block 446 len 4096 23/30 Read block 68 len 28672 24/30 Write block 561 len 36864 data: 11 11 14 12 11 14 14 17 15 25/30 Read block 155 len 4096 26/30 Flush 27/30 Write block 424 len 4096 data: 10 28/30 Read block 296 len 20480 29/30 Read block 446 len 32768 30/30 Read block 481 len 24576 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 4 3 3 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:27.938616843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1775} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 300 300 300 F F F 1 020-039 145 145 145 289 289 289 F F F 2 040-059 147 147 147 294 294 294 F F F 3 060-079 148 148 148 300 300 300 F F F 4 080-099 144 144 144 286 286 286 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 146 146 146 292 292 292 F F F 8 160-179 148 148 148 300 300 300 F F F 9 180-199 146 146 146 293 293 293 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 146 146 146 290 290 290 F F F 12 240-259 146 146 146 290 290 290 F F F 13 260-279 146 146 146 293 293 293 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 148 148 148 296 296 296 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 148 148 148 297 297 297 F F F 20 400-419 146 146 146 293 293 293 F F F 21 420-439 148 148 148 300 300 300 T T T 22 440-459 147 147 147 294 294 294 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 146 146 146 293 293 293 F F F 25 500-519 148 148 148 298 298 298 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 148 148 148 300 300 300 F F F 29 580-599 146 146 146 290 290 290 F F F Max gen: 148, Max flush: 300 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 300 300 300 F F F 1 020-039 145 145 145 289 289 289 F F F 2 040-059 147 147 147 294 294 294 F F F 3 060-079 148 148 148 300 300 300 F F F 4 080-099 144 144 144 286 286 286 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 146 146 146 292 292 292 F F F 8 160-179 148 148 148 300 300 300 F F F 9 180-199 146 146 146 293 293 293 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 146 146 146 290 290 290 F F F 12 240-259 146 146 146 290 290 290 F F F 13 260-279 146 146 146 293 293 293 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 148 148 148 296 296 296 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 148 148 148 297 297 297 F F F 20 400-419 146 146 146 293 293 293 F F F 21 420-439 149 149 149 301 301 301 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 138 138 138 276 276 276 F F F 24 480-499 146 146 146 293 293 293 F F F 25 500-519 148 148 148 298 298 298 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 143 143 143 284 284 284 F F F 28 560-579 148 148 148 300 300 300 F F F 29 580-599 146 146 146 290 290 290 F F F Max gen: 149, Max flush: 301 Begin loop 75 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.110151387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.110695356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.110716643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783} {"msg":"Crucible da8fc305-7913-4735-aeb7-c06dcaf930f4 has session id: 2429ca0e-3f3b-4d93-825c-9e0d8638040f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.110957548Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Upstairs opts: Upstairs UUID: da8fc305-7913-4735-aeb7-c06dcaf930f4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111032246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Crucible stats registered with UUID: da8fc305-7913-4735-aeb7-c06dcaf930f4","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111058442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111377955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783} {"msg":"da8fc305-7913-4735-aeb7-c06dcaf930f4 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111498597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111532102Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111560508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111585015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111606991Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111628638Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111652925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111677071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111705027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111733482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:38.111799453Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_connection connected from Ok(127.0.0.1:62964)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.111839897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:38.111866043Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:38.112001373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112031868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112060154Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_connection connected from Ok(127.0.0.1:36056)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112082951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112113356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112665954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.11269287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112708208Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.112722026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.113377289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.113405825Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.113972741Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.113998797Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.114012925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.114025163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.114038391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:38.114051559Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.113605665Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57792)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.113669595Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"io task","client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.114878906Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.114906552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.115505294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116116623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.11613804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[0]R flush_numbers[0..12]: [300, 289, 294, 300, 286, 286, 281, 292, 300, 293, 293, 290]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116153837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[0]R generation[0..12]: [148, 145, 147, 148, 144, 144, 141, 146, 148, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116168415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116182653Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[1]R flush_numbers[0..12]: [300, 289, 294, 300, 286, 286, 281, 292, 300, 293, 293, 290]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116196721Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[1]R generation[0..12]: [148, 145, 147, 148, 144, 144, 141, 146, 148, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116210629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116224777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[2]R flush_numbers[0..12]: [300, 289, 294, 300, 286, 286, 281, 292, 300, 293, 293, 290]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116238585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[2]R generation[0..12]: [148, 145, 147, 148, 144, 144, 141, 146, 148, 146, 146, 146]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116256542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.11627079Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116283768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116296876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Next flush: 302","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116309514Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116326342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"":"downstairs","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.11634017Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116352358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"0","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116365486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"1","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116378354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"client":"2","":"downstairs","session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116391452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.1164038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"da8fc305-7913-4735-aeb7-c06dcaf930f4 is now active with session: 2429ca0e-3f3b-4d93-825c-9e0d8638040f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116416608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116428686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:38:48.116440665Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 252 len 4096 data: 14 03/30 Write block 26 len 24576 data: 8 12 13 13 11 12 04/30 Write block 3 len 28672 data: 8 9 11 11 12 11 14 05/30 Read block 174 len 16384 06/30 Read block 254 len 40960 07/30 Write block 20 len 4096 data: 7 08/30 Flush 09/30 Flush 10/30 Write block 556 len 28672 data: 13 10 10 9 10 12 12 11/30 Write block 84 len 24576 data: 14 10 8 7 9 9 12/30 Write block 39 len 24576 data: 5 7 9 12 14 18 13/30 Read block 196 len 4096 14/30 Flush 15/30 Flush 16/30 Write block 382 len 40960 data: 14 11 10 9 10 8 12 16 16 15 17/30 Read block 250 len 12288 18/30 Read block 327 len 28672 19/30 Write block 267 len 20480 data: 9 8 10 9 11 20/30 Write block 231 len 16384 data: 6 9 8 7 21/30 Read block 217 len 12288 22/30 Read block 462 len 32768 23/30 Read block 460 len 32768 24/30 Read block 181 len 28672 25/30 Read block 138 len 16384 26/30 Read block 577 len 24576 27/30 Write block 477 len 20480 data: 12 15 15 16 16 28/30 Write block 51 len 12288 data: 11 12 13 29/30 Read block 503 len 32768 30/30 Write block 58 len 32768 data: 11 10 14 13 13 13 15 14 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 8 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.142843421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143370023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143404808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14346053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143486136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143508503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14353018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143556276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143578962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143601039Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143617847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143630255Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143642243Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143654201Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143666189Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143679487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143691425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143703404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143715212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14372707Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143739578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143751847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143763975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143775783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143790581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143802759Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143814987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143826916Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143838984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143850752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14386273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143874638Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143886876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143899515Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143911743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143923911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143936109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143948287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143960446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143972544Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.143984692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144003019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144023936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144037014Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144049112Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144062321Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144074739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144086647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144098755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144110953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144123081Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14413521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144147338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144159356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144171094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144182983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144194841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144206729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144221047Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144233095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144245063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144256932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14426875Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144280778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144292896Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144304905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144316873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144328691Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144340389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144352198Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144364036Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144375964Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144387912Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14439982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144411719Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144425287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144437165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144449013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144460802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14447335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144485248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144497426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144509554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144521833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144534041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144546199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144558427Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144570525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144582733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144594982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14460723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144619308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144633246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144645064Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144656913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144668691Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144680409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144692307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144704266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144716084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144727962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14473976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144751678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144763587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144775635Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144787533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14481175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144824308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144836326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144848264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144862282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.14487432Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144886319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144898617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144910815Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144923743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144935911Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144947999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144960117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144972116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144984244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.144999092Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.145019629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.145032967Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.145045085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.145057183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:38:48.145069211Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1783,"session_id":"2429ca0e-3f3b-4d93-825c-9e0d8638040f"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 303 303 303 F F F 1 020-039 150 150 150 305 305 305 F F F 2 040-059 150 150 150 305 305 305 T T T 3 060-079 148 148 148 300 300 300 T T T 4 080-099 150 150 150 305 305 305 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 146 146 146 292 292 292 F F F 8 160-179 148 148 148 300 300 300 F F F 9 180-199 146 146 146 293 293 293 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 146 146 146 290 290 290 T T T 12 240-259 150 150 150 303 303 303 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 148 148 148 296 296 296 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 150 150 150 307 307 307 F F F 20 400-419 146 146 146 293 293 293 F F F 21 420-439 149 149 149 301 301 301 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 138 138 138 276 276 276 T T T 24 480-499 146 146 146 293 293 293 T T T 25 500-519 148 148 148 298 298 298 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 F F F 28 560-579 150 150 150 305 305 305 F F F 29 580-599 146 146 146 290 290 290 F F F Max gen: 150, Max flush: 307 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 303 303 303 F F F 1 020-039 150 150 150 305 305 305 F F F 2 040-059 151 151 151 308 308 308 F F F 3 060-079 151 151 151 308 308 308 F F F 4 080-099 150 150 150 305 305 305 F F F 5 100-119 144 144 144 286 286 286 F F F 6 120-139 141 141 141 281 281 281 F F F 7 140-159 146 146 146 292 292 292 F F F 8 160-179 148 148 148 300 300 300 F F F 9 180-199 146 146 146 293 293 293 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 150 150 150 303 303 303 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 144 144 144 285 285 285 F F F 15 300-319 148 148 148 296 296 296 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 150 150 150 307 307 307 F F F 20 400-419 146 146 146 293 293 293 F F F 21 420-439 149 149 149 301 301 301 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 151 151 151 308 308 308 F F F 24 480-499 151 151 151 308 308 308 F F F 25 500-519 148 148 148 298 298 298 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 F F F 28 560-579 150 150 150 305 305 305 F F F 29 580-599 146 146 146 290 290 290 F F F Max gen: 151, Max flush: 308 Begin loop 76 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.359499762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.359884055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.35991767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791} {"msg":"Crucible 9a26e48c-3669-4784-b946-595a7a4fe5fb has session id: 9662381a-a6ed-412c-8f8e-ca26bcc231ee","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360148416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Upstairs opts: Upstairs UUID: 9a26e48c-3669-4784-b946-595a7a4fe5fb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360192269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Crucible stats registered with UUID: 9a26e48c-3669-4784-b946-595a7a4fe5fb","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360216846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360510702Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791} {"msg":"9a26e48c-3669-4784-b946-595a7a4fe5fb active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360565164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360583371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360598459Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360611937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360625125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360638123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.36065696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360680017Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360708643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360734569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:02.360878918Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:52926)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.360917682Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:02.360948987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:02.361034435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361070519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361094756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:37934)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361122462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361148798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361789743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361814109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361828207Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.361841605Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.362488169Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.362529983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.363037878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.363077272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.363105298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.363131194Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.36315833Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:02.363181066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.361906785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_connection connected from Ok(127.0.0.1:60833)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.361980714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"io task","client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.363130104Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.363167458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.363809143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364382148Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364408575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[0]R flush_numbers[0..12]: [303, 305, 308, 308, 305, 286, 281, 292, 300, 293, 293, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364424382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[0]R generation[0..12]: [150, 150, 151, 151, 150, 144, 141, 146, 148, 146, 146, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364443439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364458307Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[1]R flush_numbers[0..12]: [303, 305, 308, 308, 305, 286, 281, 292, 300, 293, 293, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364472005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[1]R generation[0..12]: [150, 150, 151, 151, 150, 144, 141, 146, 148, 146, 146, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364485823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364499801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[2]R flush_numbers[0..12]: [303, 305, 308, 308, 305, 286, 281, 292, 300, 293, 293, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364513489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[2]R generation[0..12]: [150, 150, 151, 151, 150, 144, 141, 146, 148, 146, 146, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364527117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364540845Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364553573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364566671Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364579829Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364592607Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"":"downstairs","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364605725Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364617704Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"0","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:39:12.364630782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"1","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364647549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"client":"2","":"downstairs","session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364661027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364672915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"9a26e48c-3669-4784-b946-595a7a4fe5fb is now active with session: 9662381a-a6ed-412c-8f8e-ca26bcc231ee","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364685643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.364697502Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791,"session_id":"9662381a-a6ed-412c-8f8e-ca26bcc231ee"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:12.36470957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 405 len 16384 data: 18 18 14 12 02/30 Read block 188 len 40960 03/30 Read block 356 len 36864 04/30 Read block 460 len 32768 05/30 Write block 587 len 8192 data: 10 10 06/30 Write block 429 len 20480 data: 14 15 16 17 15 07/30 Write block 134 len 12288 data: 9 11 11 08/30 Read block 121 len 8192 09/30 Write block 503 len 36864 data: 16 15 16 15 14 16 19 18 16 10/30 Write block 163 len 36864 data: 18 17 19 18 19 18 20 19 17 11/30 Read block 42 len 12288 12/30 Write block 17 len 16384 data: 7 6 7 8 13/30 Flush 14/30 Write block 107 len 16384 data: 10 10 14 13 15/30 Flush 16/30 Write block 151 len 20480 data: 11 9 11 10 10 17/30 Write block 181 len 16384 data: 9 7 8 7 18/30 Read block 276 len 28672 19/30 Write block 389 len 28672 data: 17 17 16 14 14 15 15 20/30 Write block 35 len 36864 data: 7 7 6 5 6 8 10 13 15 21/30 Read block 171 len 12288 22/30 Read block 80 len 4096 23/30 Read block 145 len 12288 24/30 Read block 242 len 28672 25/30 Write block 422 len 8192 data: 7 10 26/30 Read block 376 len 24576 27/30 Write block 281 len 20480 data: 7 7 9 10 10 28/30 Read block 534 len 20480 29/30 Write block 5 len 40960 data: 12 12 13 12 15 15 14 11 7 6 30/30 Read block 524 len 8192 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 2 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 10 Done Done Done false 31 Acked 1030 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:12.391062754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1791} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 309 309 309 T T T 1 020-039 152 152 152 311 311 311 F F F 2 040-059 152 152 152 311 311 311 F F F 3 060-079 151 151 151 308 308 308 F F F 4 080-099 150 150 150 305 305 305 F F F 5 100-119 152 152 152 310 310 310 F F F 6 120-139 152 152 152 309 309 309 F F F 7 140-159 152 152 152 311 311 311 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 152 152 152 311 311 311 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 150 150 150 303 303 303 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 144 144 144 285 285 285 T T T 15 300-319 148 148 148 296 296 296 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 152 152 152 311 311 311 F F F 20 400-419 152 152 152 309 309 309 F F F 21 420-439 152 152 152 309 309 309 T T T 22 440-459 147 147 147 294 294 294 F F F 23 460-479 151 151 151 308 308 308 F F F 24 480-499 151 151 151 308 308 308 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 F F F 28 560-579 150 150 150 305 305 305 F F F 29 580-599 152 152 152 309 309 309 F F F Max gen: 152, Max flush: 311 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 152 152 152 311 311 311 F F F 2 040-059 152 152 152 311 311 311 F F F 3 060-079 151 151 151 308 308 308 F F F 4 080-099 150 150 150 305 305 305 F F F 5 100-119 152 152 152 310 310 310 F F F 6 120-139 152 152 152 309 309 309 F F F 7 140-159 152 152 152 311 311 311 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 152 152 152 311 311 311 F F F 10 200-219 146 146 146 293 293 293 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 150 150 150 303 303 303 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 153 153 153 312 312 312 F F F 15 300-319 148 148 148 296 296 296 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 152 152 152 311 311 311 F F F 20 400-419 152 152 152 309 309 309 F F F 21 420-439 153 153 153 312 312 312 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 151 151 151 308 308 308 F F F 24 480-499 151 151 151 308 308 308 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 F F F 28 560-579 150 150 150 305 305 305 F F F 29 580-599 152 152 152 309 309 309 F F F Max gen: 153, Max flush: 312 Begin loop 77 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.584448874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.584899887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.584930922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799} {"msg":"Crucible dedd0c30-0a6a-4f82-ac2c-89b5ed435977 has session id: c6a9772e-a889-4fee-9fad-78dc5f75935d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585118025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Upstairs opts: Upstairs UUID: dedd0c30-0a6a-4f82-ac2c-89b5ed435977, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585159629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Crucible stats registered with UUID: dedd0c30-0a6a-4f82-ac2c-89b5ed435977","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585183185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585456885Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799} {"msg":"dedd0c30-0a6a-4f82-ac2c-89b5ed435977 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585546441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585564689Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585583746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585600443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585613941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585627129Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585640497Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585657585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585684841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585722645Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:26.585947052Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54651)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.585984496Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:26.586050716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45536)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.586081182Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:26.586110788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.586135794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.58616264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.58622772Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.586781988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.586813823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.586840089Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.586866656Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.587474106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.587505611Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.588073786Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.588100183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.588122519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.588143696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.588158794Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:26.588171992Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.587213284Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_connection connected from Ok(127.0.0.1:45140)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.587269016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"io task","client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.588360374Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.58838771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.58906574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589655802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589682108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[0]R flush_numbers[0..12]: [312, 311, 311, 308, 305, 310, 309, 311, 309, 311, 293, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589703335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[0]R generation[0..12]: [153, 152, 152, 151, 150, 152, 152, 152, 152, 152, 146, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589718413Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589732711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[1]R flush_numbers[0..12]: [312, 311, 311, 308, 305, 310, 309, 311, 309, 311, 293, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589746529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[1]R generation[0..12]: [153, 152, 152, 151, 150, 152, 152, 152, 152, 152, 146, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589760357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589774954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[2]R flush_numbers[0..12]: [312, 311, 311, 308, 305, 310, 309, 311, 309, 311, 293, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589788692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[2]R generation[0..12]: [153, 152, 152, 151, 150, 152, 152, 152, 152, 152, 146, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589802321Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589816219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589828877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:39:36.589841855Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Next flush: 313","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589857892Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.58987083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"":"downstairs","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589883869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589897716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"0","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589911524Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"1","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589924922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"client":"2","":"downstairs","session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.58993834Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589950519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"dedd0c30-0a6a-4f82-ac2c-89b5ed435977 is now active with session: c6a9772e-a889-4fee-9fad-78dc5f75935d","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589963487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589975435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799,"session_id":"c6a9772e-a889-4fee-9fad-78dc5f75935d"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:36.589987713Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 289 len 24576 data: 16 17 18 15 11 12 02/30 Read block 423 len 4096 03/30 Write block 463 len 24576 data: 6 6 8 10 14 10 04/30 Read block 67 len 40960 05/30 Read block 179 len 36864 06/30 Read block 383 len 12288 07/30 Read block 147 len 12288 08/30 Read block 580 len 12288 09/30 Read block 170 len 16384 10/30 Read block 381 len 40960 11/30 Read block 522 len 8192 12/30 Read block 449 len 16384 13/30 Flush 14/30 Write block 194 len 32768 data: 8 11 8 8 7 10 11 9 15/30 Write block 192 len 16384 data: 10 9 9 12 16/30 Read block 184 len 4096 17/30 Read block 538 len 28672 18/30 Write block 561 len 12288 data: 13 13 15 19/30 Write block 314 len 8192 data: 9 11 20/30 Write block 95 len 4096 data: 12 21/30 Write block 104 len 32768 data: 8 9 11 11 11 15 14 16 22/30 Read block 13 len 40960 23/30 Write block 395 len 16384 data: 16 18 17 13 24/30 Read block 98 len 28672 25/30 Read block 529 len 4096 26/30 Read block 350 len 8192 27/30 Write block 67 len 12288 data: 14 13 16 28/30 Read block 422 len 16384 29/30 Write block 135 len 40960 data: 12 12 12 11 10 9 9 11 12 13 30/30 Write block 412 len 8192 data: 12 10 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Write 1 Done Done Done false 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Read 10 Done Done Done false 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 2 Done Done Done false 28 Acked 1027 Write 3 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false 31 Acked 1030 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:36.613512996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1799} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 152 152 152 311 311 311 F F F 2 040-059 152 152 152 311 311 311 F F F 3 060-079 151 151 151 308 308 308 T T T 4 080-099 150 150 150 305 305 305 T T T 5 100-119 152 152 152 310 310 310 T T T 6 120-139 152 152 152 309 309 309 T T T 7 140-159 152 152 152 311 311 311 T T T 8 160-179 152 152 152 309 309 309 F F F 9 180-199 154 154 154 314 314 314 F F F 10 200-219 154 154 154 314 314 314 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 150 150 150 303 303 303 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 154 154 154 313 313 313 F F F 15 300-319 148 148 148 296 296 296 T T T 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 152 152 152 311 311 311 T T T 20 400-419 152 152 152 309 309 309 T T T 21 420-439 153 153 153 312 312 312 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 154 154 154 313 313 313 F F F 24 480-499 151 151 151 308 308 308 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 F F F 28 560-579 154 154 154 314 314 314 F F F 29 580-599 152 152 152 309 309 309 F F F Max gen: 154, Max flush: 314 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 152 152 152 311 311 311 F F F 2 040-059 152 152 152 311 311 311 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 155 155 155 315 315 315 F F F 6 120-139 155 155 155 315 315 315 F F F 7 140-159 155 155 155 315 315 315 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 154 154 154 314 314 314 F F F 10 200-219 154 154 154 314 314 314 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 150 150 150 303 303 303 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 154 154 154 313 313 313 F F F 15 300-319 155 155 155 315 315 315 F F F 16 320-339 148 148 148 296 296 296 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 155 155 155 315 315 315 F F F 21 420-439 153 153 153 312 312 312 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 154 154 154 313 313 313 F F F 24 480-499 151 151 151 308 308 308 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 F F F 28 560-579 154 154 154 314 314 314 F F F 29 580-599 152 152 152 309 309 309 F F F Max gen: 155, Max flush: 315 Begin loop 78 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.861834934Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862378623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862409938Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807} {"msg":"Crucible 6935dbb4-1adf-4036-8cc1-4476d7068577 has session id: 2cdd46f8-2969-4065-880f-31a9a5218f11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862431895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Upstairs opts: Upstairs UUID: 6935dbb4-1adf-4036-8cc1-4476d7068577, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862481358Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Crucible stats registered with UUID: 6935dbb4-1adf-4036-8cc1-4476d7068577","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862506164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862833696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807} {"msg":"6935dbb4-1adf-4036-8cc1-4476d7068577 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862901426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862927202Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.86294333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862957487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.862980734Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.86300577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863031216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863045604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863059542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.86307349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_connection connected from Ok(127.0.0.1:48879)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.86327827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_connection connected from Ok(127.0.0.1:45646)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863349669Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:50.863379165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:50.863405561Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:39:50.863431327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863451694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863473691Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.863585444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.864094509Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.864118335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.864136562Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.864150831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.86476053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.864787376Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.865295441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.865317118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.865330685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.865342744Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.865355922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:39:50.86536901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.864039887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_connection connected from Ok(127.0.0.1:58781)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.864124175Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"io task","client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.865152272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.865193266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.865771231Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866366942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866400697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[0]R flush_numbers[0..12]: [312, 311, 311, 315, 315, 315, 315, 315, 309, 314, 314, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866417764Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[0]R generation[0..12]: [153, 152, 152, 155, 155, 155, 155, 155, 152, 154, 154, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866432322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.86644782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[1]R flush_numbers[0..12]: [312, 311, 311, 315, 315, 315, 315, 315, 309, 314, 314, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866462848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[1]R generation[0..12]: [153, 152, 152, 155, 155, 155, 155, 155, 152, 154, 154, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866477186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866491444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[2]R flush_numbers[0..12]: [312, 311, 311, 315, 315, 315, 315, 315, 309, 314, 314, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866505391Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[2]R generation[0..12]: [153, 152, 152, 155, 155, 155, 155, 155, 152, 154, 154, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866533067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:40:00.866545716Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866560783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866573901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866589219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"":"downstairs","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866602577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866614486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"0","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866627624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"1","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866640762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"client":"2","":"downstairs","session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866653869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866665898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"6935dbb4-1adf-4036-8cc1-4476d7068577 is now active with session: 2cdd46f8-2969-4065-880f-31a9a5218f11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866678766Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866690504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807,"session_id":"2cdd46f8-2969-4065-880f-31a9a5218f11"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:00.866702512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 325 len 32768 data: 12 13 11 12 10 10 10 11 02/30 Write block 130 len 40960 data: 10 10 10 10 10 13 13 13 12 11 03/30 Read block 283 len 4096 04/30 Read block 296 len 12288 05/30 Read block 31 len 40960 06/30 Read block 555 len 8192 07/30 Write block 110 len 28672 data: 15 17 19 22 18 16 15 08/30 Write block 424 len 16384 data: 11 11 12 14 09/30 Read block 299 len 4096 10/30 Read block 45 len 4096 11/30 Read block 376 len 4096 12/30 Read block 526 len 12288 13/30 Read block 585 len 28672 14/30 Read block 467 len 12288 15/30 Write block 199 len 20480 data: 11 12 10 12 11 16/30 Write block 243 len 8192 data: 8 9 17/30 Write block 109 len 20480 data: 16 16 18 20 23 18/30 Read block 222 len 4096 19/30 Flush 20/30 Write block 135 len 36864 data: 14 14 14 13 12 10 10 12 13 21/30 Read block 266 len 4096 22/30 Read block 13 len 36864 23/30 Read block 347 len 24576 24/30 Read block 500 len 36864 25/30 Flush 26/30 Write block 473 len 32768 data: 8 8 10 13 13 16 16 17 27/30 Write block 29 len 20480 data: 14 12 13 11 11 28/30 Write block 542 len 16384 data: 5 7 6 7 29/30 Write block 411 len 12288 data: 14 13 11 30/30 Write block 573 len 40960 data: 18 17 17 15 11 12 10 14 15 12 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 8 Done Done Done false 28 Acked 1027 Write 5 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Write 3 Done Done Done false 31 Acked 1030 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:00.891698837Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1807} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 152 152 152 311 311 311 T T T 2 040-059 152 152 152 311 311 311 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 156 156 156 317 317 317 F F F 7 140-159 156 156 156 317 317 317 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 156 156 156 316 316 316 F F F 10 200-219 156 156 156 316 316 316 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 156 156 156 316 316 316 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 154 154 154 313 313 313 F F F 15 300-319 155 155 155 315 315 315 F F F 16 320-339 156 156 156 316 316 316 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 155 155 155 315 315 315 T T T 21 420-439 156 156 156 316 316 316 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 154 154 154 313 313 313 T T T 24 480-499 151 151 151 308 308 308 T T T 25 500-519 152 152 152 309 309 309 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 150 150 150 305 305 305 T T T 28 560-579 154 154 154 314 314 314 T T T 29 580-599 152 152 152 309 309 309 T T T Max gen: 156, Max flush: 317 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 152 152 152 311 311 311 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 156 156 156 317 317 317 F F F 7 140-159 156 156 156 317 317 317 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 156 156 156 316 316 316 F F F 10 200-219 156 156 156 316 316 316 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 156 156 156 316 316 316 F F F 13 260-279 150 150 150 307 307 307 F F F 14 280-299 154 154 154 313 313 313 F F F 15 300-319 155 155 155 315 315 315 F F F 16 320-339 156 156 156 316 316 316 F F F 17 340-359 141 141 141 281 281 281 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 157 157 157 318 318 318 F F F 21 420-439 156 156 156 316 316 316 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 157 157 157 318 318 318 F F F 24 480-499 157 157 157 318 318 318 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 144 144 144 287 287 287 F F F 27 540-559 157 157 157 318 318 318 F F F 28 560-579 157 157 157 318 318 318 F F F 29 580-599 157 157 157 318 318 318 F F F Max gen: 157, Max flush: 318 Begin loop 79 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.140073057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.140471178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.140501093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815} {"msg":"Crucible f0955a10-7e05-4cfd-9fd7-bd3f12df72df has session id: 6b26a414-2d5d-40af-afb8-cce0f5562c6a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.140787101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Upstairs opts: Upstairs UUID: f0955a10-7e05-4cfd-9fd7-bd3f12df72df, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.140829755Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Crucible stats registered with UUID: f0955a10-7e05-4cfd-9fd7-bd3f12df72df","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.140852811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141179133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815} {"msg":"f0955a10-7e05-4cfd-9fd7-bd3f12df72df active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.14120246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141216617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141231275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141244973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141258651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141275679Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141289337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141302505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141316043Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141342489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:15.141630776Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:15.141756997Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:65446)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141783423Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:15.141809519Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141829756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141850953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:36747)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.14187223Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.141894217Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.142426098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.142452714Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.142472601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.142487499Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143144292Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143172197Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143683222Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143709568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143724146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143736634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.143749752Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:15.14376271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.142989695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43712)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.143175747Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"io task","client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.144211583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.144239399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.144831622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.14545279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145480486Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[0]R flush_numbers[0..12]: [312, 318, 311, 315, 315, 316, 317, 317, 309, 316, 316, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145497053Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[0]R generation[0..12]: [153, 157, 152, 155, 155, 156, 156, 156, 152, 156, 156, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145511581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145525509Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[1]R flush_numbers[0..12]: [312, 318, 311, 315, 315, 316, 317, 317, 309, 316, 316, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145539277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[1]R generation[0..12]: [153, 157, 152, 155, 155, 156, 156, 156, 152, 156, 156, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145553085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145567393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[2]R flush_numbers[0..12]: [312, 318, 311, 315, 315, 316, 317, 317, 309, 316, 316, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145581551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[2]R generation[0..12]: [153, 157, 152, 155, 155, 156, 156, 156, 152, 156, 156, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145595389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145609216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,Wait for a query_work_queue command to finish before sending IO "":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145626124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145639392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Next flush: 319","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.14565213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145668568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"":"downstairs","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145682086Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145693874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"0","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145707122Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"1","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.14572019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"client":"2","":"downstairs","session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145733528Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145745466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"f0955a10-7e05-4cfd-9fd7-bd3f12df72df is now active with session: 6b26a414-2d5d-40af-afb8-cce0f5562c6a","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145758184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145770552Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815,"session_id":"6b26a414-2d5d-40af-afb8-cce0f5562c6a"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:25.145783001Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 521 len 36864 data: 13 11 12 11 10 9 10 9 8 02/30 Read block 351 len 40960 03/30 Read block 407 len 4096 04/30 Read block 508 len 28672 05/30 Read block 238 len 24576 06/30 Read block 434 len 24576 07/30 Read block 495 len 16384 08/30 Flush 09/30 Write block 254 len 28672 data: 15 10 12 13 15 13 10 10/30 Read block 15 len 16384 11/30 Write block 330 len 24576 data: 11 11 12 14 13 10 12/30 Read block 55 len 40960 13/30 Flush 14/30 Write block 295 len 40960 data: 11 11 12 11 14 14 12 12 12 12 15/30 Write block 286 len 28672 data: 12 15 14 17 18 19 16 16/30 Write block 539 len 12288 data: 7 7 4 17/30 Flush 18/30 Flush 19/30 Write block 324 len 40960 data: 12 13 14 12 13 11 12 12 13 15 20/30 Write block 44 len 32768 data: 19 14 15 15 11 11 10 12 21/30 Write block 348 len 20480 data: 11 11 11 11 12 22/30 Write block 133 len 36864 data: 11 11 15 15 15 14 13 11 11 23/30 Read block 146 len 40960 24/30 Flush 25/30 Write block 310 len 32768 data: 11 12 11 11 10 12 11 10 26/30 Read block 212 len 12288 27/30 Read block 528 len 12288 28/30 Read block 582 len 4096 29/30 Write block 347 len 4096 data: 10 30/30 Read block 307 len 32768 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 26 Acked 1025 Write 8 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 3 Done Done Done false 29 Acked 1028 Read 1 Done Done Done false 30 Acked 1029 Write 1 Done Done Done false 31 Acked 1030 Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:25.177286801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1815} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 158 158 158 324 324 324 F F F 7 140-159 158 158 158 324 324 324 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 156 156 156 316 316 316 F F F 10 200-219 156 156 156 316 316 316 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 158 158 158 320 320 320 F F F 13 260-279 158 158 158 320 320 320 F F F 14 280-299 158 158 158 322 322 322 F F F 15 300-319 158 158 158 321 321 321 T T T 16 320-339 158 158 158 324 324 324 F F F 17 340-359 158 158 158 324 324 324 T T T 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 157 157 157 318 318 318 F F F 21 420-439 156 156 156 316 316 316 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 157 157 157 318 318 318 F F F 24 480-499 157 157 157 318 318 318 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 158 158 158 322 322 322 F F F 27 540-559 158 158 158 322 322 322 F F F 28 560-579 157 157 157 318 318 318 F F F 29 580-599 157 157 157 318 318 318 F F F Max gen: 158, Max flush: 324 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 153 153 153 312 312 312 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 158 158 158 324 324 324 F F F 7 140-159 158 158 158 324 324 324 F F F 8 160-179 152 152 152 309 309 309 F F F 9 180-199 156 156 156 316 316 316 F F F 10 200-219 156 156 156 316 316 316 F F F 11 220-239 151 151 151 308 308 308 F F F 12 240-259 158 158 158 320 320 320 F F F 13 260-279 158 158 158 320 320 320 F F F 14 280-299 158 158 158 322 322 322 F F F 15 300-319 159 159 159 325 325 325 F F F 16 320-339 158 158 158 324 324 324 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 157 157 157 318 318 318 F F F 21 420-439 156 156 156 316 316 316 F F F 22 440-459 147 147 147 294 294 294 F F F 23 460-479 157 157 157 318 318 318 F F F 24 480-499 157 157 157 318 318 318 F F F 25 500-519 152 152 152 309 309 309 F F F 26 520-539 158 158 158 322 322 322 F F F 27 540-559 158 158 158 322 322 322 F F F 28 560-579 157 157 157 318 318 318 F F F 29 580-599 157 157 157 318 318 318 F F F Max gen: 159, Max flush: 325 Begin loop 80 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.496401446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497050659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497082685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823} {"msg":"Crucible a5b6197f-9f48-487d-975d-785717e335a2 has session id: 58c23a56-0fe1-4309-a1cd-27d26f7b8b4c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497106301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Upstairs opts: Upstairs UUID: a5b6197f-9f48-487d-975d-785717e335a2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.49718301Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Crucible stats registered with UUID: a5b6197f-9f48-487d-975d-785717e335a2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497203037Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497473897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823} {"msg":"a5b6197f-9f48-487d-975d-785717e335a2 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497618965Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497637483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497652121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497665479Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497687545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.49772002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497747176Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497773143Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.49778876Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497814006Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:55905)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.497997349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59379)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498035534Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:39.498266539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:39.498307513Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:39.498337099Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498361865Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498387132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498418447Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498769455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498793581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498903155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.498932011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.499376925Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.499534581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.499941241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.499964228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.499978006Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.500004242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.50001805Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:39.500031218Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.499537601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39270)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.49968076Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"io task","client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.500715926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.500851416Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.501523407Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502090383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502113999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[0]R flush_numbers[0..12]: [312, 318, 324, 315, 315, 316, 324, 324, 309, 316, 316, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502128857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[0]R generation[0..12]: [153, 157, 158, 155, 155, 156, 158, 158, 152, 156, 156, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502143205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502157383Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[1]R flush_numbers[0..12]: [312, 318, 324, 315, 315, 316, 324, 324, 309, 316, 316, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502171191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[1]R generation[0..12]: [153, 157, 158, 155, 155, 156, 158, 158, 152, 156, 156, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502184989Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502198987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[2]R flush_numbers[0..12]: [312, 318, 324, 315, 315, 316, 324, 324, 309, 316, 316, 308]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502212594Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[2]R generation[0..12]: [153, 157, 158, 155, 155, 156, 158, 158, 152, 156, 156, 151]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502226212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502240221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502253079Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502269426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Next flush: 326","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502282994Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502299152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"":"downstairs","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.50231294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502325108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"0","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502338386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"1","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502351264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"client":"2","":"downstairs","session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502364342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.50237659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"a5b6197f-9f48-487d-975d-785717e335a2 is now active with session: 58c23a56-0fe1-4309-a1cd-27d26f7b8b4c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502389468Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502401386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823,"session_id":"58c23a56-0fe1-4309-a1cd-27d26f7b8b4c"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:40:49.502413305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 5 len 40960 data: 13 13 14 13 16 16 15 12 8 7 02/30 Write block 486 len 36864 data: 19 16 19 19 16 13 12 12 16 03/30 Flush 04/30 Write block 283 len 20480 data: 10 11 11 13 16 05/30 Read block 63 len 16384 06/30 Read block 168 len 24576 07/30 Read block 55 len 32768 08/30 Read block 48 len 4096 09/30 Write block 233 len 32768 data: 9 8 6 7 7 9 10 10 10/30 Read block 49 len 16384 11/30 Flush 12/30 Write block 156 len 40960 data: 13 14 13 15 17 15 19 19 18 20 13/30 Read block 350 len 12288 14/30 Read block 164 len 20480 15/30 Read block 504 len 20480 16/30 Write block 172 len 28672 data: 14 13 13 14 9 10 8 17/30 Write block 483 len 40960 data: 19 22 20 20 17 20 20 17 14 13 18/30 Flush 19/30 Write block 407 len 8192 data: 15 13 20/30 Read block 308 len 40960 21/30 Read block 247 len 28672 22/30 Write block 440 len 8192 data: 8 9 23/30 Read block 431 len 4096 24/30 Read block 427 len 4096 25/30 Write block 517 len 36864 data: 14 14 12 13 14 12 13 12 11 26/30 Write block 268 len 36864 data: 9 11 10 12 10 13 14 16 16 27/30 Write block 513 len 20480 data: 13 17 18 16 15 28/30 Read block 307 len 32768 29/30 Write block 561 len 8192 data: 14 14 30/30 Write block 540 len 40960 data: 8 5 6 8 7 8 8 9 9 8 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 19 Acked 1018 Write 2 Done Done Done false 20 Acked 1019 Read 10 Done Done Done false 21 Acked 1020 Read 7 Done Done Done false 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Read 1 Done Done Done false 24 Acked 1023 Read 1 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 9 Done Done Done false 27 Acked 1026 Write 5 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Write 2 Done Done Done false 30 Acked 1029 Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:40:49.53030986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1823} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 326 326 326 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 158 158 158 324 324 324 F F F 7 140-159 160 160 160 328 328 328 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 156 156 156 316 316 316 F F F 10 200-219 156 156 156 316 316 316 F F F 11 220-239 160 160 160 327 327 327 F F F 12 240-259 160 160 160 327 327 327 F F F 13 260-279 158 158 158 320 320 320 T T T 14 280-299 160 160 160 327 327 327 F F F 15 300-319 159 159 159 325 325 325 F F F 16 320-339 158 158 158 324 324 324 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 157 157 157 318 318 318 T T T 21 420-439 156 156 156 316 316 316 F F F 22 440-459 147 147 147 294 294 294 T T T 23 460-479 157 157 157 318 318 318 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 152 152 152 309 309 309 T T T 26 520-539 158 158 158 322 322 322 T T T 27 540-559 158 158 158 322 322 322 T T T 28 560-579 157 157 157 318 318 318 T T T 29 580-599 157 157 157 318 318 318 F F F Max gen: 160, Max flush: 328 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 326 326 326 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 158 158 158 324 324 324 F F F 7 140-159 160 160 160 328 328 328 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 156 156 156 316 316 316 F F F 10 200-219 156 156 156 316 316 316 F F F 11 220-239 160 160 160 327 327 327 F F F 12 240-259 160 160 160 327 327 327 F F F 13 260-279 161 161 161 329 329 329 F F F 14 280-299 160 160 160 327 327 327 F F F 15 300-319 159 159 159 325 325 325 F F F 16 320-339 158 158 158 324 324 324 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 161 161 161 329 329 329 F F F 21 420-439 156 156 156 316 316 316 F F F 22 440-459 161 161 161 329 329 329 F F F 23 460-479 157 157 157 318 318 318 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 161 161 161 329 329 329 F F F 26 520-539 161 161 161 329 329 329 F F F 27 540-559 161 161 161 329 329 329 F F F 28 560-579 161 161 161 329 329 329 F F F 29 580-599 157 157 157 318 318 318 F F F Max gen: 161, Max flush: 329 Begin loop 81 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.774087282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.774468525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.774496541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831} {"msg":"Crucible 02256b08-ba0d-4ae9-a0f3-5dd7b8430ad1 has session id: 18c56955-792e-4481-b26f-4bf9daa35e40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.774727957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Upstairs opts: Upstairs UUID: 02256b08-ba0d-4ae9-a0f3-5dd7b8430ad1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.77477093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Crucible stats registered with UUID: 02256b08-ba0d-4ae9-a0f3-5dd7b8430ad1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.774796397Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775151814Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831} {"msg":"02256b08-ba0d-4ae9-a0f3-5dd7b8430ad1 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.77517709Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775199787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775226123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775253769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775278715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775303601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775326248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775349225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775373041Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775396278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_connection connected from Ok(127.0.0.1:40806)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775487034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:03.775528778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:03.775562303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:03.775638942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775667008Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775689884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_connection connected from Ok(127.0.0.1:63230)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775713161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.775844011Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.776356385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.77639006Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.776417586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.776443353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.776888147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777004259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777391322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777430346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777458582Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777518263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777544699Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:03.777559867Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.777005339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_connection connected from Ok(127.0.0.1:45073)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.777097905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"io task","client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.778059283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.778090269Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.778724494Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779297959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779325505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[0]R flush_numbers[0..12]: [326, 318, 324, 315, 315, 316, 324, 328, 328, 316, 316, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779342073Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[0]R generation[0..12]: [160, 157, 158, 155, 155, 156, 158, 160, 160, 156, 156, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779356781Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779371759Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[1]R flush_numbers[0..12]: [326, 318, 324, 315, 315, 316, 324, 328, 328, 316, 316, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779385626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[1]R generation[0..12]: [160, 157, 158, 155, 155, 156, 158, 160, 160, 156, 156, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779399474Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779413602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[2]R flush_numbers[0..12]: [326, 318, 324, 315, 315, 316, 324, 328, 328, 316, 316, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.77942722Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[2]R generation[0..12]: [160, 157, 158, 155, 155, 156, 158, 160, 160, 156, 156, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779441178Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-08-05T19:41:13.779455096Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779471724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779491681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779515007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779539414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"":"downstairs","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779556141Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779568209Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"0","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779581248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"1","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779594126Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"client":"2","":"downstairs","session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779607233Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779619212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"02256b08-ba0d-4ae9-a0f3-5dd7b8430ad1 is now active with session: 18c56955-792e-4481-b26f-4bf9daa35e40","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.77963201Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779643678Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831,"session_id":"18c56955-792e-4481-b26f-4bf9daa35e40"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:13.779655926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 280 len 16384 data: 9 8 8 11 02/30 Read block 119 len 36864 03/30 Write block 536 len 8192 data: 8 7 04/30 Write block 126 len 4096 data: 7 05/30 Read block 440 len 12288 06/30 Write block 240 len 16384 data: 11 9 10 9 07/30 Write block 188 len 20480 data: 6 7 7 9 11 08/30 Read block 221 len 20480 09/30 Write block 285 len 24576 data: 12 14 17 15 18 19 10/30 Write block 127 len 36864 data: 9 7 9 11 11 11 12 12 16 11/30 Read block 499 len 28672 12/30 Write block 464 len 32768 data: 7 9 11 15 11 7 7 6 13/30 Read block 438 len 32768 14/30 Read block 376 len 8192 15/30 Read block 596 len 8192 16/30 Read block 467 len 8192 17/30 Flush 18/30 Write block 249 len 12288 data: 10 11 13 19/30 Write block 205 len 16384 data: 17 18 20 17 20/30 Read block 378 len 24576 21/30 Write block 129 len 32768 data: 10 12 12 12 13 13 17 16 22/30 Write block 593 len 8192 data: 8 8 23/30 Read block 535 len 32768 24/30 Write block 596 len 16384 data: 9 8 5 2 25/30 Write block 469 len 4096 data: 8 26/30 Read block 53 len 20480 27/30 Write block 273 len 32768 data: 14 15 17 17 16 13 10 10 28/30 Read block 319 len 16384 29/30 Write block 246 len 8192 data: 8 7 30/30 Read block 588 len 16384 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 8 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Write 1 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 8 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 2 Done Done Done false 31 Acked 1030 Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:13.803366091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1831} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 326 326 326 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 162 162 162 330 330 330 T T T 7 140-159 160 160 160 328 328 328 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 162 162 162 330 330 330 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 160 160 160 327 327 327 F F F 12 240-259 162 162 162 331 331 331 T T T 13 260-279 161 161 161 329 329 329 T T T 14 280-299 162 162 162 330 330 330 T T T 15 300-319 159 159 159 325 325 325 F F F 16 320-339 158 158 158 324 324 324 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 161 161 161 329 329 329 F F F 21 420-439 156 156 156 316 316 316 F F F 22 440-459 161 161 161 329 329 329 F F F 23 460-479 162 162 162 330 330 330 T T T 24 480-499 160 160 160 328 328 328 F F F 25 500-519 161 161 161 329 329 329 F F F 26 520-539 162 162 162 330 330 330 F F F 27 540-559 161 161 161 329 329 329 F F F 28 560-579 161 161 161 329 329 329 F F F 29 580-599 157 157 157 318 318 318 T T T Max gen: 162, Max flush: 331 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 160 160 160 326 326 326 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 155 155 155 315 315 315 F F F 4 080-099 155 155 155 315 315 315 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 163 163 163 332 332 332 F F F 7 140-159 160 160 160 328 328 328 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 162 162 162 330 330 330 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 160 160 160 327 327 327 F F F 12 240-259 163 163 163 332 332 332 F F F 13 260-279 163 163 163 332 332 332 F F F 14 280-299 163 163 163 332 332 332 F F F 15 300-319 159 159 159 325 325 325 F F F 16 320-339 158 158 158 324 324 324 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 155 155 155 315 315 315 F F F 20 400-419 161 161 161 329 329 329 F F F 21 420-439 156 156 156 316 316 316 F F F 22 440-459 161 161 161 329 329 329 F F F 23 460-479 163 163 163 332 332 332 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 161 161 161 329 329 329 F F F 26 520-539 162 162 162 330 330 330 F F F 27 540-559 161 161 161 329 329 329 F F F 28 560-579 161 161 161 329 329 329 F F F 29 580-599 163 163 163 332 332 332 F F F Max gen: 163, Max flush: 332 Begin loop 82 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.037781871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.03853037Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038563555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839} {"msg":"Crucible 68d82199-98d2-4da3-b9c9-37e4ce0c2917 has session id: e96713a4-0b64-4790-99bd-990562de9bf2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038587062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Upstairs opts: Upstairs UUID: 68d82199-98d2-4da3-b9c9-37e4ce0c2917, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038622007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Crucible stats registered with UUID: 68d82199-98d2-4da3-b9c9-37e4ce0c2917","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038647333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038880098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839} {"msg":"68d82199-98d2-4da3-b9c9-37e4ce0c2917 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038907924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038928241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038943869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038957607Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038971125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.038984273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039007629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039030886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039046044Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039060222Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:28.039256902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:54803)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039293477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:28.039324092Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:28.039454633Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039482579Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039507475Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58789)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039535841Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.039561847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.040039426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.040072981Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.040196993Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.040228668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.040562949Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.040678452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.04109316Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.041121126Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.041136704Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.04116248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.041176988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:28.041190366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.040442926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56345)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.040611421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"io task","client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.041606244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.041640899Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.04217015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042720419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042748485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[0]R flush_numbers[0..12]: [326, 318, 324, 315, 315, 316, 332, 328, 328, 330, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042765372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[0]R generation[0..12]: [160, 157, 158, 155, 155, 156, 163, 160, 160, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.04278034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042794758Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[1]R flush_numbers[0..12]: [326, 318, 324, 315, 315, 316, 332, 328, 328, 330, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042808626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[1]R generation[0..12]: [160, 157, 158, 155, 155, 156, 163, 160, 160, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042822634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042836962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[2]R flush_numbers[0..12]: [326, 318, 324, 315, 315, 316, 332, 328, 328, 330, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.04285058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[2]R generation[0..12]: [160, 157, 158, 155, 155, 156, 163, 160, 160, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042864327Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042878336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042898433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.04291278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042925888Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042938927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"":"downstairs","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042952604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042964663Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"0","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042977901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"1","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.042990849Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"client":"2","":"downstairs","session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.043003927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.043016155Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"68d82199-98d2-4da3-b9c9-37e4ce0c2917 is now active with session: e96713a4-0b64-4790-99bd-990562de9bf2","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.043029023Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.043040921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839,"session_id":"e96713a4-0b64-4790-99bd-990562de9bf2"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:38.04305309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 180 len 40960 data: 8 10 8 9 8 7 7 7 7 8 02/30 Read block 454 len 20480 03/30 Write block 183 len 36864 data: 10 9 8 8 8 8 9 8 10 04/30 Read block 246 len 20480 05/30 Write block 432 len 24576 data: 18 16 15 12 14 14 06/30 Read block 522 len 24576 07/30 Write block 399 len 20480 data: 12 15 15 16 15 08/30 Write block 93 len 4096 data: 11 09/30 Read block 517 len 40960 10/30 Read block 428 len 20480 11/30 Read block 229 len 12288 12/30 Write block 145 len 4096 data: 15 13/30 Write block 97 len 12288 data: 12 13 12 14/30 Write block 513 len 40960 data: 14 18 19 17 16 15 13 14 15 13 15/30 Write block 276 len 24576 data: 18 17 14 11 11 9 16/30 Write block 11 len 28672 data: 16 13 9 8 9 8 8 17/30 Write block 68 len 36864 data: 14 17 10 8 8 7 6 5 8 18/30 Write block 293 len 40960 data: 12 13 12 12 13 12 15 15 13 13 19/30 Write block 89 len 24576 data: 10 9 7 8 12 10 20/30 Read block 339 len 16384 21/30 Read block 303 len 12288 22/30 Write block 318 len 24576 data: 9 8 8 7 8 10 23/30 Read block 93 len 32768 24/30 Read block 207 len 24576 25/30 Read block 406 len 4096 26/30 Read block 77 len 16384 27/30 Read block 153 len 36864 28/30 Write block 575 len 32768 data: 18 16 12 13 11 15 16 13 29/30 Read block 100 len 16384 30/30 Read block 376 len 12288 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 6 Done Done Done false 24 Acked 1023 Read 8 Done Done Done false 25 Acked 1024 Read 6 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 4 Done Done Done false 28 Acked 1027 Read 9 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false 31 Acked 1030 Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:38.069545283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1839} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 333 333 333 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 164 164 164 333 333 333 F F F 4 080-099 164 164 164 333 333 333 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 163 163 163 332 332 332 F F F 7 140-159 164 164 164 333 333 333 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 164 164 164 333 333 333 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 160 160 160 327 327 327 F F F 12 240-259 163 163 163 332 332 332 F F F 13 260-279 164 164 164 333 333 333 F F F 14 280-299 164 164 164 333 333 333 F F F 15 300-319 164 164 164 333 333 333 T T T 16 320-339 158 158 158 324 324 324 T T T 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 164 164 164 333 333 333 F F F 20 400-419 164 164 164 333 333 333 F F F 21 420-439 164 164 164 333 333 333 F F F 22 440-459 161 161 161 329 329 329 F F F 23 460-479 163 163 163 332 332 332 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 164 164 164 333 333 333 F F F 26 520-539 164 164 164 333 333 333 F F F 27 540-559 161 161 161 329 329 329 F F F 28 560-579 161 161 161 329 329 329 T T T 29 580-599 163 163 163 332 332 332 T T T Max gen: 164, Max flush: 333 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 333 333 333 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 158 158 158 324 324 324 F F F 3 060-079 164 164 164 333 333 333 F F F 4 080-099 164 164 164 333 333 333 F F F 5 100-119 156 156 156 316 316 316 F F F 6 120-139 163 163 163 332 332 332 F F F 7 140-159 164 164 164 333 333 333 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 164 164 164 333 333 333 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 160 160 160 327 327 327 F F F 12 240-259 163 163 163 332 332 332 F F F 13 260-279 164 164 164 333 333 333 F F F 14 280-299 164 164 164 333 333 333 F F F 15 300-319 165 165 165 334 334 334 F F F 16 320-339 165 165 165 334 334 334 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 F F F 19 380-399 164 164 164 333 333 333 F F F 20 400-419 164 164 164 333 333 333 F F F 21 420-439 164 164 164 333 333 333 F F F 22 440-459 161 161 161 329 329 329 F F F 23 460-479 163 163 163 332 332 332 F F F 24 480-499 160 160 160 328 328 328 F F F 25 500-519 164 164 164 333 333 333 F F F 26 520-539 164 164 164 333 333 333 F F F 27 540-559 161 161 161 329 329 329 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 165 165 165 334 334 334 F F F Max gen: 165, Max flush: 334 Begin loop 83 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.279718236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.27996255Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.279978697Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847} {"msg":"Crucible 45484838-305d-4952-b93d-bc0558d71213 has session id: 2f7096f0-cfcb-4915-8f76-df0d392f2715","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280490791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Upstairs opts: Upstairs UUID: 45484838-305d-4952-b93d-bc0558d71213, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280523366Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Crucible stats registered with UUID: 45484838-305d-4952-b93d-bc0558d71213","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280537914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280823932Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847} {"msg":"45484838-305d-4952-b93d-bc0558d71213 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280943984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280961511Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280975999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.280989447Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281002915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281016093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.28103973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281060897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281075025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281088683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_connection connected from Ok(127.0.0.1:59474)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281350574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_connection connected from Ok(127.0.0.1:41426)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281642481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:52.281745135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:52.281770191Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:41:52.281786439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281808226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281834172Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.281867947Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.282143406Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.282167943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.28218173Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.282197818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.282783271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.282841873Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.283427726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.28346408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.283486917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.283500615Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.283514153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:41:52.283527341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.283221196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_connection connected from Ok(127.0.0.1:43342)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.283388481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"io task","client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.284345899Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.284374575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.284934812Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285477521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285505467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[0]R flush_numbers[0..12]: [333, 318, 324, 333, 333, 316, 332, 333, 328, 333, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285522015Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[0]R generation[0..12]: [164, 157, 158, 164, 164, 156, 163, 164, 160, 164, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285536732Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.28555091Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[1]R flush_numbers[0..12]: [333, 318, 324, 333, 333, 316, 332, 333, 328, 333, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285564918Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[1]R generation[0..12]: [164, 157, 158, 164, 164, 156, 163, 164, 160, 164, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285578806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285592754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[2]R flush_numbers[0..12]: [333, 318, 324, 333, 333, 316, 332, 333, 328, 333, 331, 327]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285606352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[2]R generation[0..12]: [164, 157, 158, 164, 164, 156, 163, 164, 160, 164, 162, 160]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285624999Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:42:02.285640297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285656895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285670193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Next flush: 335","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285682951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285695729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"":"downstairs","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285708667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285720505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"0","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285733723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"1","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285746701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"client":"2","":"downstairs","session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285759769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285771778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"45484838-305d-4952-b93d-bc0558d71213 is now active with session: 2f7096f0-cfcb-4915-8f76-df0d392f2715","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285784316Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285796194Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847,"session_id":"2f7096f0-cfcb-4915-8f76-df0d392f2715"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:02.285808242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 518 len 40960 data: 16 14 15 16 14 14 13 12 10 11 02/30 Read block 421 len 16384 03/30 Flush 04/30 Write block 314 len 20480 data: 11 13 12 11 10 05/30 Write block 398 len 36864 data: 14 13 16 16 17 16 15 19 19 06/30 Write block 404 len 24576 data: 16 20 20 16 14 15 07/30 Read block 299 len 32768 08/30 Write block 104 len 32768 data: 9 10 12 12 12 17 17 19 09/30 Read block 337 len 24576 10/30 Flush 11/30 Write block 490 len 4096 data: 18 12/30 Write block 451 len 32768 data: 7 7 8 7 7 6 5 5 13/30 Read block 314 len 20480 14/30 Flush 15/30 Flush 16/30 Write block 56 len 32768 data: 12 13 12 11 15 14 14 14 17/30 Read block 63 len 40960 18/30 Read block 99 len 24576 19/30 Read block 17 len 32768 20/30 Flush 21/30 Flush 22/30 Flush 23/30 Write block 365 len 12288 data: 12 15 20 24/30 Write block 236 len 36864 data: 8 8 10 11 12 10 11 10 10 25/30 Write block 457 len 16384 data: 6 6 6 6 26/30 Read block 6 len 16384 27/30 Read block 267 len 24576 28/30 Write block 452 len 24576 data: 8 9 8 8 7 7 29/30 Write block 533 len 40960 data: 10 13 12 9 8 5 8 9 6 7 30/30 Write block 99 len 8192 data: 13 10 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Write 9 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 4 Done Done Done false 27 Acked 1026 Read 6 Done Done Done false 28 Acked 1027 Write 6 Done Done Done false 29 Acked 1028 Write 10 Sent Sent Sent false 30 Acked 1029 Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:02.312931352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1847} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 333 333 333 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 F F F 3 060-079 166 166 166 339 339 339 F F F 4 080-099 164 164 164 333 333 333 T T T 5 100-119 166 166 166 336 336 336 T T T 6 120-139 163 163 163 332 332 332 F F F 7 140-159 164 164 164 333 333 333 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 164 164 164 333 333 333 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 160 160 160 327 327 327 T T T 12 240-259 163 163 163 332 332 332 T T T 13 260-279 164 164 164 333 333 333 F F F 14 280-299 164 164 164 333 333 333 F F F 15 300-319 166 166 166 336 336 336 F F F 16 320-339 165 165 165 334 334 334 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 148 148 148 295 295 295 T T T 19 380-399 166 166 166 336 336 336 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 164 164 164 333 333 333 F F F 22 440-459 166 166 166 337 337 337 T T T 23 460-479 163 163 163 332 332 332 T T T 24 480-499 166 166 166 337 337 337 F F F 25 500-519 166 166 166 335 335 335 F F F 26 520-539 166 166 166 335 335 335 T T T 27 540-559 161 161 161 329 329 329 T T T 28 560-579 165 165 165 334 334 334 F F F 29 580-599 165 165 165 334 334 334 F F F Max gen: 166, Max flush: 339 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 333 333 333 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 F F F 3 060-079 166 166 166 339 339 339 F F F 4 080-099 167 167 167 340 340 340 F F F 5 100-119 167 167 167 340 340 340 F F F 6 120-139 163 163 163 332 332 332 F F F 7 140-159 164 164 164 333 333 333 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 164 164 164 333 333 333 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 167 167 167 340 340 340 F F F 12 240-259 167 167 167 340 340 340 F F F 13 260-279 164 164 164 333 333 333 F F F 14 280-299 164 164 164 333 333 333 F F F 15 300-319 166 166 166 336 336 336 F F F 16 320-339 165 165 165 334 334 334 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 167 167 167 340 340 340 F F F 19 380-399 166 166 166 336 336 336 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 164 164 164 333 333 333 F F F 22 440-459 167 167 167 340 340 340 F F F 23 460-479 167 167 167 340 340 340 F F F 24 480-499 166 166 166 337 337 337 F F F 25 500-519 166 166 166 335 335 335 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 165 165 165 334 334 334 F F F Max gen: 167, Max flush: 340 Begin loop 84 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.585049613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.585401121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.585431127Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855} {"msg":"Crucible b70e2572-676f-45a3-8ebd-d516a16ff250 has session id: f464124f-f899-4e4f-8355-084ba0ea0610","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.585696847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Upstairs opts: Upstairs UUID: b70e2572-676f-45a3-8ebd-d516a16ff250, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.585796973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Crucible stats registered with UUID: b70e2572-676f-45a3-8ebd-d516a16ff250","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.585828648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586179446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855} {"msg":"b70e2572-676f-45a3-8ebd-d516a16ff250 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586259264Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586279901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586305757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586330823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586360219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586384575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586411202Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586427109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586446446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586468183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_connection connected from Ok(127.0.0.1:34546)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.58662563Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_connection connected from Ok(127.0.0.1:58719)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.586679322Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:16.586788395Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:16.586829969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:16.586982287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.587015692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.587042168Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.587110868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.587445778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.587470634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.587485462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.58749921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588076645Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588204806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588634672Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588661458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588675946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588700292Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.58871417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:16.588731378Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.587947163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_connection connected from Ok(127.0.0.1:51398)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.588018773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"io task","client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.589081145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.58911964Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.589654381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590218687Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590246533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[0]R flush_numbers[0..12]: [333, 318, 339, 339, 340, 340, 332, 333, 328, 333, 331, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[0]R generation[0..12]: [164, 157, 166, 166, 167, 167, 163, 164, 160, 164, 162, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590277988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590292096Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[1]R flush_numbers[0..12]: [333, 318, 339, 339, 340, 340, 332, 333, 328, 333, 331, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590305874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[1]R generation[0..12]: [164, 157, 166, 166, 167, 167, 163, 164, 160, 164, 162, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590319472Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.59033337Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[2]R flush_numbers[0..12]: [333, 318, 339, 339, 340, 340, 332, 333, 328, 333, 331, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590347228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[2]R generation[0..12]: [164, 157, 166, 166, 167, 167, 163, 164, 160, 164, 162, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590365555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590380083Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.59039729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590410818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590424126Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590437304Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"":"downstairs","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590450872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590462731Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"0","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590476399Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"1","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590489617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"client":"2","":"downstairs","session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590502785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590514773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"b70e2572-676f-45a3-8ebd-d516a16ff250 is now active with session: f464124f-f899-4e4f-8355-084ba0ea0610","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590527421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590539149Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855,"session_id":"f464124f-f899-4e4f-8355-084ba0ea0610"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:26.590551128Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 439 len 40960 data: 11 9 10 12 13 10 10 9 7 9 02/30 Read block 449 len 16384 03/30 Write block 589 len 12288 data: 10 12 10 04/30 Write block 69 len 40960 data: 18 11 9 9 8 7 6 9 11 13 05/30 Read block 424 len 36864 06/30 Write block 0 len 28672 data: 3 4 6 9 10 14 14 07/30 Write block 297 len 12288 data: 14 13 16 08/30 Write block 140 len 4096 data: 12 09/30 Write block 145 len 4096 data: 16 10/30 Read block 412 len 36864 11/30 Flush 12/30 Write block 515 len 16384 data: 20 18 17 17 13/30 Read block 254 len 32768 14/30 Read block 345 len 8192 15/30 Write block 374 len 24576 data: 14 14 11 13 13 16 16/30 Flush 17/30 Write block 10 len 36864 data: 17 17 14 10 9 10 9 9 7 18/30 Read block 480 len 36864 19/30 Write block 281 len 36864 data: 10 9 12 12 13 15 18 16 19 20/30 Flush 21/30 Write block 254 len 8192 data: 16 11 22/30 Write block 284 len 40960 data: 13 14 16 19 17 20 20 20 17 13 23/30 Read block 38 len 20480 24/30 Write block 468 len 16384 data: 12 9 8 7 25/30 Read block 295 len 4096 26/30 Read block 566 len 36864 27/30 Write block 81 len 4096 data: 13 28/30 Write block 260 len 24576 data: 11 11 10 9 9 13 29/30 Read block 322 len 40960 30/30 Write block 437 len 32768 data: 15 13 12 10 11 13 14 11 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 22 Acked 1021 Write 2 Done Done Done false 23 Acked 1022 Write 10 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 4 Done Done Done false 26 Acked 1025 Read 1 Done Done Done false 27 Acked 1026 Read 9 Done Done Done false 28 Acked 1027 Write 1 Done Done Done false 29 Acked 1028 Write 6 Done Done Done false 30 Acked 1029 Read 10 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:26.617151695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1855} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 343 343 343 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 F F F 3 060-079 168 168 168 341 341 341 F F F 4 080-099 167 167 167 340 340 340 T T T 5 100-119 167 167 167 340 340 340 F F F 6 120-139 163 163 163 332 332 332 F F F 7 140-159 168 168 168 341 341 341 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 164 164 164 333 333 333 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 167 167 167 340 340 340 F F F 12 240-259 167 167 167 340 340 340 T T T 13 260-279 164 164 164 333 333 333 T T T 14 280-299 168 168 168 344 344 344 T T T 15 300-319 166 166 166 336 336 336 F F F 16 320-339 165 165 165 334 334 334 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 168 168 168 342 342 342 F F F 19 380-399 166 166 166 336 336 336 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 168 168 168 341 341 341 T T T 22 440-459 168 168 168 341 341 341 T T T 23 460-479 167 167 167 340 340 340 T T T 24 480-499 166 166 166 337 337 337 F F F 25 500-519 168 168 168 342 342 342 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 168 168 168 341 341 341 F F F Max gen: 168, Max flush: 344 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 343 343 343 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 F F F 3 060-079 168 168 168 341 341 341 F F F 4 080-099 169 169 169 345 345 345 F F F 5 100-119 167 167 167 340 340 340 F F F 6 120-139 163 163 163 332 332 332 F F F 7 140-159 168 168 168 341 341 341 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 164 164 164 333 333 333 F F F 10 200-219 162 162 162 331 331 331 F F F 11 220-239 167 167 167 340 340 340 F F F 12 240-259 169 169 169 345 345 345 F F F 13 260-279 169 169 169 345 345 345 F F F 14 280-299 169 169 169 345 345 345 F F F 15 300-319 166 166 166 336 336 336 F F F 16 320-339 165 165 165 334 334 334 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 168 168 168 342 342 342 F F F 19 380-399 166 166 166 336 336 336 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 169 169 169 345 345 345 F F F 22 440-459 169 169 169 345 345 345 F F F 23 460-479 169 169 169 345 345 345 F F F 24 480-499 166 166 166 337 337 337 F F F 25 500-519 168 168 168 342 342 342 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 168 168 168 341 341 341 F F F Max gen: 169, Max flush: 345 Begin loop 85 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.862585581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863007048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863037303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863} {"msg":"Crucible 68556122-03a1-49be-9e26-e8af1a7ad927 has session id: 6062256f-7134-440a-82c1-ab595a7d61d3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.86326116Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Upstairs opts: Upstairs UUID: 68556122-03a1-49be-9e26-e8af1a7ad927, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863309133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Crucible stats registered with UUID: 68556122-03a1-49be-9e26-e8af1a7ad927","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863336319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863690477Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863} {"msg":"68556122-03a1-49be-9e26-e8af1a7ad927 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863721422Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863745258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863769465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863792842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863816998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863841504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.86386832Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863894596Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863910084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.863924142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:40.864117273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:40.864158277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37068)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.864188623Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:40.864216789Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.864239485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.864262902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:43016)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.864287148Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.864313144Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.864982025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.865007602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.865021959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.865035527Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.865664914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.86569392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.866280403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.866306669Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.866321167Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.866334815Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.866351942Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:40.86636557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.865196154Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:44265)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.865253165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"io task","client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.866247238Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.86636495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867034161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867586089Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867622544Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[0]R flush_numbers[0..12]: [343, 318, 339, 341, 345, 340, 332, 341, 328, 333, 331, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.86764846Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[0]R generation[0..12]: [168, 157, 166, 168, 169, 167, 163, 168, 160, 164, 162, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867663648Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867677936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[1]R flush_numbers[0..12]: [343, 318, 339, 341, 345, 340, 332, 341, 328, 333, 331, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867692024Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[1]R generation[0..12]: [168, 157, 166, 168, 169, 167, 163, 168, 160, 164, 162, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867705872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867724309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[2]R flush_numbers[0..12]: [343, 318, 339, 341, 345, 340, 332, 341, 328, 333, 331, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867739247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[2]R generation[0..12]: [168, 157, 166, 168, 169, 167, 163, 168, 160, 164, 162, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867753305Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2024-08-05T19:42:50.867771572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.86778537Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867798478Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Next flush: 346","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867811326Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867823914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"":"downstairs","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867836872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867848741Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"0","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867861739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"1","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867874767Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"client":"2","":"downstairs","session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867887775Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867899813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"68556122-03a1-49be-9e26-e8af1a7ad927 is now active with session: 6062256f-7134-440a-82c1-ab595a7d61d3","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867912641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867927649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863,"session_id":"6062256f-7134-440a-82c1-ab595a7d61d3"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:42:50.867939887Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 324 len 24576 data: 13 14 15 13 14 12 03/30 Flush 04/30 Write block 470 len 32768 data: 9 8 6 9 9 11 14 14 05/30 Read block 208 len 36864 06/30 Write block 181 len 8192 data: 11 9 07/30 Read block 98 len 36864 08/30 Write block 314 len 40960 data: 12 14 13 12 11 9 9 8 9 11 09/30 Read block 255 len 28672 10/30 Read block 463 len 32768 11/30 Read block 21 len 28672 12/30 Read block 196 len 20480 13/30 Read block 186 len 8192 14/30 Write block 591 len 28672 data: 11 8 9 9 11 10 9 15/30 Write block 257 len 36864 data: 14 16 14 12 12 11 10 10 14 16/30 Write block 203 len 24576 data: 12 12 18 19 21 18 17/30 Write block 88 len 40960 data: 10 11 10 8 9 13 11 13 13 13 18/30 Write block 461 len 20480 data: 6 7 7 8 10 19/30 Write block 142 len 16384 data: 13 14 14 17 20/30 Read block 44 len 4096 21/30 Write block 323 len 28672 data: 12 14 15 16 14 15 13 22/30 Write block 243 len 32768 data: 11 11 9 9 8 6 11 12 23/30 Flush 24/30 Write block 313 len 16384 data: 12 13 15 14 25/30 Read block 456 len 36864 26/30 Read block 34 len 36864 27/30 Write block 121 len 24576 data: 11 11 12 9 9 8 28/30 Read block 61 len 4096 29/30 Read block 200 len 36864 30/30 Read block 565 len 20480 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 4 Done Done Done false 25 Acked 1024 Read 9 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Write 6 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 9 Done Done Done false 30 Acked 1029 Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 6 6 7 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:42:50.896008007Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1863} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 343 343 343 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 F F F 3 060-079 168 168 168 341 341 341 F F F 4 080-099 170 170 170 348 348 348 F F F 5 100-119 167 167 167 340 340 340 F F F 6 120-139 163 163 163 332 332 332 T T T 7 140-159 170 170 170 348 348 348 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 170 170 170 348 348 348 F F F 10 200-219 170 170 170 348 348 348 F F F 11 220-239 167 167 167 340 340 340 F F F 12 240-259 170 170 170 348 348 348 F F F 13 260-279 170 170 170 348 348 348 F F F 14 280-299 169 169 169 345 345 345 F F F 15 300-319 170 170 170 348 348 348 T T T 16 320-339 170 170 170 348 348 348 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 168 168 168 342 342 342 F F F 19 380-399 166 166 166 336 336 336 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 169 169 169 345 345 345 F F F 22 440-459 169 169 169 345 345 345 F F F 23 460-479 170 170 170 348 348 348 F F F 24 480-499 166 166 166 337 337 337 F F F 25 500-519 168 168 168 342 342 342 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 170 170 170 348 348 348 F F F Max gen: 170, Max flush: 348 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 343 343 343 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 F F F 3 060-079 168 168 168 341 341 341 F F F 4 080-099 170 170 170 348 348 348 F F F 5 100-119 167 167 167 340 340 340 F F F 6 120-139 171 171 171 349 349 349 F F F 7 140-159 170 170 170 348 348 348 F F F 8 160-179 160 160 160 328 328 328 F F F 9 180-199 170 170 170 348 348 348 F F F 10 200-219 170 170 170 348 348 348 F F F 11 220-239 167 167 167 340 340 340 F F F 12 240-259 170 170 170 348 348 348 F F F 13 260-279 170 170 170 348 348 348 F F F 14 280-299 169 169 169 345 345 345 F F F 15 300-319 171 171 171 349 349 349 F F F 16 320-339 170 170 170 348 348 348 F F F 17 340-359 159 159 159 325 325 325 F F F 18 360-379 168 168 168 342 342 342 F F F 19 380-399 166 166 166 336 336 336 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 169 169 169 345 345 345 F F F 22 440-459 169 169 169 345 345 345 F F F 23 460-479 170 170 170 348 348 348 F F F 24 480-499 166 166 166 337 337 337 F F F 25 500-519 168 168 168 342 342 342 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 170 170 170 348 348 348 F F F Max gen: 171, Max flush: 349 Begin loop 86 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.080530132Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.080978705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081003542Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871} {"msg":"Crucible 94296257-8dea-4eac-a3be-f617bf357d16 has session id: dae15bd3-1122-4ea5-99e3-a31a39707f27","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.08135404Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Upstairs opts: Upstairs UUID: 94296257-8dea-4eac-a3be-f617bf357d16, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081401673Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Crucible stats registered with UUID: 94296257-8dea-4eac-a3be-f617bf357d16","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081426329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081727704Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871} {"msg":"94296257-8dea-4eac-a3be-f617bf357d16 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081840028Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081860165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081876232Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.08189065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081904628Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081918506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081931884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081944952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.0819587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.081973188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_connection connected from Ok(127.0.0.1:63230)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082146642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_connection connected from Ok(127.0.0.1:46988)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082322986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:05.082367029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:05.082394845Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:05.082421621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082441909Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082463495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082484902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082965701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.082994027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.083019433Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.083044759Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.083553254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.083688973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.084024604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.08404983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.084065138Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.084207617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.084227194Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:05.084240902Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.084173162Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_connection connected from Ok(127.0.0.1:41654)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.08432183Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"io task","client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.085211308Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.085239414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.085775854Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086301576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086329372Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[0]R flush_numbers[0..12]: [343, 318, 339, 341, 348, 340, 349, 348, 328, 348, 348, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086346119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[0]R generation[0..12]: [168, 157, 166, 168, 170, 167, 171, 170, 160, 170, 170, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086360877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086375255Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[1]R flush_numbers[0..12]: [343, 318, 339, 341, 348, 340, 349, 348, 328, 348, 348, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086389323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[1]R generation[0..12]: [168, 157, 166, 168, 170, 167, 171, 170, 160, 170, 170, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.08640823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086423018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[2]R flush_numbers[0..12]: [343, 318, 339, 341, 348, 340, 349, 348, 328, 348, 348, 340]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086437026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[2]R generation[0..12]: [168, 157, 166, 168, 170, 167, 171, 170, 160, 170, 170, 167]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086450644Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086468532Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086482139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086495328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086508166Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086520994Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"":"downstairs","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086534442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.08654645Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"0","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086559618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"1","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086572836Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"client":"2","":"downstairs","session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086587004Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086599152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"94296257-8dea-4eac-a3be-f617bf357d16 is now active with session: dae15bd3-1122-4ea5-99e3-a31a39707f27","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.08661443Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086626448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871,"session_id":"dae15bd3-1122-4ea5-99e3-a31a39707f27"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:15.086638626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 389 len 40960 data: 18 18 17 15 15 16 17 19 18 15 02/30 Read block 191 len 8192 03/30 Write block 61 len 28672 data: 15 15 15 16 15 14 15 04/30 Read block 61 len 12288 05/30 Write block 165 len 12288 data: 21 19 20 06/30 Flush 07/30 Write block 258 len 28672 data: 17 15 13 13 12 11 11 08/30 Write block 97 len 28672 data: 14 14 14 11 10 9 9 09/30 Write block 435 len 20480 data: 13 15 16 14 13 10/30 Write block 91 len 20480 data: 9 10 14 12 14 11/30 Flush 12/30 Write block 175 len 32768 data: 15 10 11 9 8 9 12 10 13/30 Flush 14/30 Write block 279 len 4096 data: 12 15/30 Flush 16/30 Flush 17/30 Write block 509 len 32768 data: 20 19 17 16 15 19 21 19 18/30 Write block 443 len 20480 data: 15 12 11 10 8 19/30 Write block 339 len 40960 data: 11 10 10 10 10 10 14 14 11 12 20/30 Read block 463 len 20480 21/30 Write block 222 len 36864 data: 10 8 5 6 5 5 6 7 6 22/30 Write block 280 len 20480 data: 12 11 10 13 14 23/30 Read block 429 len 8192 24/30 Write block 382 len 32768 data: 15 12 11 10 11 9 13 19 25/30 Write block 163 len 36864 data: 20 19 22 20 21 19 21 20 18 26/30 Write block 81 len 12288 data: 14 14 15 27/30 Write block 56 len 28672 data: 13 14 13 12 16 16 16 28/30 Write block 483 len 16384 data: 20 23 21 21 29/30 Write block 202 len 16384 data: 13 13 13 19 30/30 Write block 253 len 36864 data: 16 17 12 13 15 18 16 14 14 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 17 Acked 1016 Write 8 Done Done Done false 18 Acked 1017 Write 5 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Write 9 Done Done Done false 22 Acked 1021 Write 5 Done Done Done false 23 Acked 1022 Read 2 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 4 Done Done Done false 30 Acked 1029 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:15.115102617Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1871} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 343 343 343 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 166 166 166 339 339 339 T T T 3 060-079 172 172 172 350 350 350 T T T 4 080-099 172 172 172 351 351 351 T T T 5 100-119 172 172 172 351 351 351 F F F 6 120-139 171 171 171 349 349 349 F F F 7 140-159 170 170 170 348 348 348 F F F 8 160-179 172 172 172 352 352 352 T T T 9 180-199 172 172 172 352 352 352 F F F 10 200-219 170 170 170 348 348 348 T T T 11 220-239 167 167 167 340 340 340 T T T 12 240-259 172 172 172 351 351 351 T T T 13 260-279 172 172 172 353 353 353 T T T 14 280-299 169 169 169 345 345 345 T T T 15 300-319 171 171 171 349 349 349 F F F 16 320-339 170 170 170 348 348 348 T T T 17 340-359 159 159 159 325 325 325 T T T 18 360-379 168 168 168 342 342 342 F F F 19 380-399 172 172 172 350 350 350 T T T 20 400-419 166 166 166 336 336 336 F F F 21 420-439 172 172 172 351 351 351 F F F 22 440-459 169 169 169 345 345 345 T T T 23 460-479 170 170 170 348 348 348 F F F 24 480-499 166 166 166 337 337 337 T T T 25 500-519 168 168 168 342 342 342 T T T 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 170 170 170 348 348 348 F F F Max gen: 172, Max flush: 353 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 168 168 168 343 343 343 F F F 1 020-039 157 157 157 318 318 318 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 172 172 172 351 351 351 F F F 6 120-139 171 171 171 349 349 349 F F F 7 140-159 170 170 170 348 348 348 F F F 8 160-179 173 173 173 354 354 354 F F F 9 180-199 172 172 172 352 352 352 F F F 10 200-219 173 173 173 354 354 354 F F F 11 220-239 173 173 173 354 354 354 F F F 12 240-259 173 173 173 354 354 354 F F F 13 260-279 173 173 173 354 354 354 F F F 14 280-299 173 173 173 354 354 354 F F F 15 300-319 171 171 171 349 349 349 F F F 16 320-339 173 173 173 354 354 354 F F F 17 340-359 173 173 173 354 354 354 F F F 18 360-379 168 168 168 342 342 342 F F F 19 380-399 173 173 173 354 354 354 F F F 20 400-419 166 166 166 336 336 336 F F F 21 420-439 172 172 172 351 351 351 F F F 22 440-459 173 173 173 354 354 354 F F F 23 460-479 170 170 170 348 348 348 F F F 24 480-499 173 173 173 354 354 354 F F F 25 500-519 173 173 173 354 354 354 F F F 26 520-539 167 167 167 340 340 340 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 F F F 29 580-599 170 170 170 348 348 348 F F F Max gen: 173, Max flush: 354 Begin loop 87 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.456071733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.456649487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.456686652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879} {"msg":"Crucible b763db4d-e119-4e5b-8687-57509eb43fb1 has session id: 3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.456752622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Upstairs opts: Upstairs UUID: b763db4d-e119-4e5b-8687-57509eb43fb1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.456795336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Crucible stats registered with UUID: b763db4d-e119-4e5b-8687-57509eb43fb1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.456821342Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457155012Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879} {"msg":"b763db4d-e119-4e5b-8687-57509eb43fb1 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457220553Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457250488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457266866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457286463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457311289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457343174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.45736878Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457392607Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457411934Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.45744031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_connection connected from Ok(127.0.0.1:48140)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457682454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_connection connected from Ok(127.0.0.1:33644)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457721788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:29.457835621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:29.457863487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:29.457881124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457894133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.457907471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.458129408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.458609687Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.458636083Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.45865121Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.458665068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459289806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.4593297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459818068Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459850583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459876179Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459891087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459904665Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:29.459917943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.459443033Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_connection connected from Ok(127.0.0.1:37265)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.459628266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"io task","client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.460548209Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.460591793Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461231208Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461799094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.46182544Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[0]R flush_numbers[0..12]: [343, 318, 354, 354, 354, 351, 349, 348, 354, 352, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461841877Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[0]R generation[0..12]: [168, 157, 173, 173, 173, 172, 171, 170, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461856505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461875163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[1]R flush_numbers[0..12]: [343, 318, 354, 354, 354, 351, 349, 348, 354, 352, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.46188937Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[1]R generation[0..12]: [168, 157, 173, 173, 173, 172, 171, 170, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461903239Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461917216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[2]R flush_numbers[0..12]: [343, 318, 354, 354, 354, 351, 349, 348, 354, 352, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461930914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[2]R generation[0..12]: [168, 157, 173, 173, 173, 172, 171, 170, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461944572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.46195869Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461971778Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {Wait for a query_work_queue command to finish before sending IO "msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.461988676Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Next flush: 355","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462002224Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462015102Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"":"downstairs","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.46202799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462039928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"0","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462053246Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"1","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462068504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"client":"2","":"downstairs","session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462082102Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.46209428Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"b763db4d-e119-4e5b-8687-57509eb43fb1 is now active with session: 3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462106978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462118696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879,"session_id":"3b775ffa-7dd7-4c25-bfe9-aa88a3aeec98"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:39.462130645Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 4096 data: 18 02/30 Write block 361 len 40960 data: 11 9 11 11 13 16 21 19 17 16 03/30 Read block 83 len 16384 04/30 Flush 05/30 Write block 19 len 32768 data: 8 9 8 10 12 11 6 9 06/30 Read block 249 len 24576 07/30 Read block 246 len 4096 08/30 Read block 311 len 20480 09/30 Write block 486 len 40960 data: 22 18 21 21 19 15 14 13 17 15 10/30 Flush 11/30 Write block 279 len 40960 data: 13 13 12 11 14 15 15 17 20 18 12/30 Read block 332 len 20480 13/30 Read block 355 len 40960 14/30 Read block 359 len 20480 15/30 Read block 91 len 32768 16/30 Write block 534 len 20480 data: 14 13 10 9 6 17/30 Write block 10 len 12288 data: 18 18 15 18/30 Write block 111 len 36864 data: 20 21 24 19 17 16 11 13 12 19/30 Write block 258 len 8192 data: 19 17 20/30 Read block 225 len 40960 21/30 Read block 196 len 32768 22/30 Write block 130 len 8192 data: 13 13 23/30 Write block 397 len 32768 data: 19 16 14 17 17 18 17 17 24/30 Read block 325 len 16384 25/30 Read block 468 len 36864 26/30 Read block 78 len 20480 27/30 Write block 242 len 16384 data: 12 12 12 10 28/30 Write block 575 len 40960 data: 19 17 13 14 12 16 17 14 9 9 29/30 Read block 454 len 20480 30/30 Write block 338 len 28672 data: 11 12 11 11 11 11 11 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Read 8 Done Done Done false 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Write 9 Done Done Done false 20 Acked 1019 Write 2 Done Done Done false 21 Acked 1020 Read 10 Done Done Done false 22 Acked 1021 Read 8 Done Done Done false 23 Acked 1022 Write 2 Done Done Done false 24 Acked 1023 Write 8 Done Done Done false 25 Acked 1024 Read 4 Done Done Done false 26 Acked 1025 Read 9 Done Done Done false 27 Acked 1026 Read 5 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Write 10 Done Done Done false 30 Acked 1029 Read 5 Done Done Done false 31 Acked 1030 Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:39.488144559Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1879} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 356 356 356 T T T 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 172 172 172 351 351 351 T T T 6 120-139 171 171 171 349 349 349 T T T 7 140-159 170 170 170 348 348 348 F F F 8 160-179 173 173 173 354 354 354 F F F 9 180-199 172 172 172 352 352 352 F F F 10 200-219 173 173 173 354 354 354 F F F 11 220-239 173 173 173 354 354 354 F F F 12 240-259 173 173 173 354 354 354 T T T 13 260-279 174 174 174 357 357 357 F F F 14 280-299 174 174 174 357 357 357 F F F 15 300-319 171 171 171 349 349 349 F F F 16 320-339 173 173 173 354 354 354 T T T 17 340-359 173 173 173 354 354 354 T T T 18 360-379 174 174 174 355 355 355 F F F 19 380-399 173 173 173 354 354 354 T T T 20 400-419 166 166 166 336 336 336 T T T 21 420-439 172 172 172 351 351 351 F F F 22 440-459 173 173 173 354 354 354 F F F 23 460-479 170 170 170 348 348 348 F F F 24 480-499 174 174 174 356 356 356 F F F 25 500-519 173 173 173 354 354 354 F F F 26 520-539 167 167 167 340 340 340 T T T 27 540-559 167 167 167 340 340 340 F F F 28 560-579 165 165 165 334 334 334 T T T 29 580-599 170 170 170 348 348 348 T T T Max gen: 174, Max flush: 357 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 175 175 175 358 358 358 F F F 6 120-139 175 175 175 358 358 358 F F F 7 140-159 170 170 170 348 348 348 F F F 8 160-179 173 173 173 354 354 354 F F F 9 180-199 172 172 172 352 352 352 F F F 10 200-219 173 173 173 354 354 354 F F F 11 220-239 173 173 173 354 354 354 F F F 12 240-259 175 175 175 358 358 358 F F F 13 260-279 174 174 174 357 357 357 F F F 14 280-299 174 174 174 357 357 357 F F F 15 300-319 171 171 171 349 349 349 F F F 16 320-339 175 175 175 358 358 358 F F F 17 340-359 175 175 175 358 358 358 F F F 18 360-379 174 174 174 355 355 355 F F F 19 380-399 175 175 175 358 358 358 F F F 20 400-419 175 175 175 358 358 358 F F F 21 420-439 172 172 172 351 351 351 F F F 22 440-459 173 173 173 354 354 354 F F F 23 460-479 170 170 170 348 348 348 F F F 24 480-499 174 174 174 356 356 356 F F F 25 500-519 173 173 173 354 354 354 F F F 26 520-539 175 175 175 358 358 358 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 175 175 175 358 358 358 F F F 29 580-599 175 175 175 358 358 358 F F F Max gen: 175, Max flush: 358 Begin loop 88 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.775148174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.775621743Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.775654089Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887} {"msg":"Crucible f8bf262e-79c3-472b-a7e7-2238aeb924ab has session id: ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.77584687Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Upstairs opts: Upstairs UUID: f8bf262e-79c3-472b-a7e7-2238aeb924ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.775889864Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Crucible stats registered with UUID: f8bf262e-79c3-472b-a7e7-2238aeb924ab","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.77591275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776289484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887} {"msg":"f8bf262e-79c3-472b-a7e7-2238aeb924ab active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776323419Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776337087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776351345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776364753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776378071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776391149Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776408077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.77645547Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776479836Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776496664Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:53.776678927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:53.7767241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50323)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776752516Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:43:53.776779512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776801408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776824125Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.776875737Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41589)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.777024275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.777429555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.777456891Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.777543338Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.777569435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.777932761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778044054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778399782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778438806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778466632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778526373Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778552279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:43:53.778566567Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.778068591Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63532)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.778136251Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"io task","client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.779192414Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.779233608Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.779869864Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.78043394Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780462726Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[0]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 358, 348, 354, 352, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780477984Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[0]R generation[0..12]: [175, 174, 173, 173, 173, 175, 175, 170, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780492562Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780507119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[1]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 358, 348, 354, 352, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780521097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[1]R generation[0..12]: [175, 174, 173, 173, 173, 175, 175, 170, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780534935Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780548853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[2]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 358, 348, 354, 352, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780562761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[2]R generation[0..12]: [175, 174, 173, 173, 173, 175, 175, 170, 173, 172, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780576229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780590277Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780603275Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780616463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pidWait for a query_work_queue command to finish before sending IO ":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Next flush: 359","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780633651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780646539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"":"downstairs","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780662336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780674705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"0","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780688063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"1","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780701161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"client":"2","":"downstairs","session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780714379Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780726357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"f8bf262e-79c3-472b-a7e7-2238aeb924ab is now active with session: ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780739045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780750923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887,"session_id":"ba7362f9-b30f-42e4-a6fa-ee271e5eaf8f"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:03.780762881Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 468 len 36864 data: 13 10 10 9 7 10 10 12 15 02/30 Read block 429 len 40960 03/30 Read block 313 len 32768 04/30 Flush 05/30 Flush 06/30 Write block 337 len 32768 data: 13 12 13 12 12 12 12 12 07/30 Write block 573 len 24576 data: 19 18 20 18 14 15 08/30 Read block 312 len 28672 09/30 Read block 368 len 20480 10/30 Write block 187 len 16384 data: 9 9 10 9 11/30 Flush 12/30 Write block 477 len 16384 data: 15 17 17 19 13/30 Write block 436 len 32768 data: 16 17 15 14 11 12 14 16 14/30 Read block 113 len 24576 15/30 Write block 248 len 28672 data: 7 12 13 14 15 17 18 16/30 Write block 159 len 12288 data: 16 18 16 17/30 Read block 542 len 16384 18/30 Read block 283 len 28672 19/30 Read block 438 len 28672 20/30 Write block 250 len 24576 data: 14 15 16 18 19 13 21/30 Read block 104 len 20480 22/30 Write block 413 len 32768 data: 12 10 9 10 8 6 6 9 23/30 Read block 258 len 28672 24/30 Write block 265 len 28672 data: 15 9 10 10 12 11 13 25/30 Read block 112 len 12288 26/30 Read block 475 len 28672 27/30 Write block 484 len 40960 data: 24 22 23 19 22 22 20 16 15 14 28/30 Read block 270 len 16384 29/30 Write block 434 len 12288 data: 16 14 17 30/30 Read block 11 len 24576 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 18 Acked 1017 Read 4 Done Done Done false 19 Acked 1018 Read 7 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Write 6 Done Done Done false 22 Acked 1021 Read 5 Done Done Done false 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 7 Done Done Done false 25 Acked 1024 Write 7 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 4 Done Done Done false 30 Acked 1029 Write 3 Done Done Done false 31 Acked 1030 Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 0 2 Done 13 13 14 40 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:03.80957784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1887} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 175 175 175 358 358 358 F F F 6 120-139 175 175 175 358 358 358 F F F 7 140-159 176 176 176 362 362 362 F F F 8 160-179 176 176 176 362 362 362 F F F 9 180-199 176 176 176 361 361 361 F F F 10 200-219 173 173 173 354 354 354 F F F 11 220-239 173 173 173 354 354 354 F F F 12 240-259 176 176 176 362 362 362 T T T 13 260-279 174 174 174 357 357 357 T T T 14 280-299 174 174 174 357 357 357 F F F 15 300-319 171 171 171 349 349 349 F F F 16 320-339 176 176 176 361 361 361 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 174 174 174 355 355 355 F F F 19 380-399 175 175 175 358 358 358 F F F 20 400-419 175 175 175 358 358 358 T T T 21 420-439 176 176 176 362 362 362 T T T 22 440-459 176 176 176 362 362 362 F F F 23 460-479 176 176 176 362 362 362 F F F 24 480-499 176 176 176 362 362 362 T T T 25 500-519 173 173 173 354 354 354 F F F 26 520-539 175 175 175 358 358 358 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 176 176 176 361 361 361 F F F 29 580-599 175 175 175 358 358 358 F F F Max gen: 176, Max flush: 362 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 175 175 175 358 358 358 F F F 6 120-139 175 175 175 358 358 358 F F F 7 140-159 176 176 176 362 362 362 F F F 8 160-179 176 176 176 362 362 362 F F F 9 180-199 176 176 176 361 361 361 F F F 10 200-219 173 173 173 354 354 354 F F F 11 220-239 173 173 173 354 354 354 F F F 12 240-259 177 177 177 363 363 363 F F F 13 260-279 177 177 177 363 363 363 F F F 14 280-299 174 174 174 357 357 357 F F F 15 300-319 171 171 171 349 349 349 F F F 16 320-339 176 176 176 361 361 361 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 174 174 174 355 355 355 F F F 19 380-399 175 175 175 358 358 358 F F F 20 400-419 177 177 177 363 363 363 F F F 21 420-439 177 177 177 363 363 363 F F F 22 440-459 176 176 176 362 362 362 F F F 23 460-479 176 176 176 362 362 362 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 173 173 173 354 354 354 F F F 26 520-539 175 175 175 358 358 358 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 176 176 176 361 361 361 F F F 29 580-599 175 175 175 358 358 358 F F F Max gen: 177, Max flush: 363 Begin loop 89 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.02867543Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.029253314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.029285579Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895} {"msg":"Crucible ceddb860-218f-4d50-b709-f9e7addf9bd9 has session id: e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.029530393Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Upstairs opts: Upstairs UUID: ceddb860-218f-4d50-b709-f9e7addf9bd9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.029649735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Crucible stats registered with UUID: ceddb860-218f-4d50-b709-f9e7addf9bd9","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.029671802Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030068093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895} {"msg":"ceddb860-218f-4d50-b709-f9e7addf9bd9 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.03009184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030105118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030119646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030133174Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030151441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030164839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030178357Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030200474Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.03022575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030241768Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:18.030478442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_connection connected from Ok(127.0.0.1:33400)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030515417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:18.030666285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:18.030705799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030728685Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030751712Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_connection connected from Ok(127.0.0.1:48519)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030776118Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.030793946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.031269525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.0313021Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.031328236Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.031356192Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032010645Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032084135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032603098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032628844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032644061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032683695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032700813Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:18.032714241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.031930917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_connection connected from Ok(127.0.0.1:53392)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.03218142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"io task","client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.033074228Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.033101893Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.033647093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034245284Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.03427298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[0]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 358, 362, 362, 361, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034289828Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[0]R generation[0..12]: [175, 174, 173, 173, 173, 175, 175, 176, 176, 176, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034304715Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034318953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[1]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 358, 362, 362, 361, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034332651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[1]R generation[0..12]: [175, 174, 173, 173, 173, 175, 175, 176, 176, 176, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034346389Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034360297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[2]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 358, 362, 362, 361, 354, 354]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034374035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[2]R generation[0..12]: [175, 174, 173, 173, 173, 175, 175, 176, 176, 176, 173, 173]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034387513Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.03440579Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034419018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034435806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Next flush: 364","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034449014Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034461962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"":"downstairs","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.03447501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034489578Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"0","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034506055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"1","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034522093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"client":"2","":"downstairs","session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034538371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034554008Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"ceddb860-218f-4d50-b709-f9e7addf9bd9 is now active with session: e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034569996Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034584804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895,"session_id":"e9f6bad7-fe2a-4c02-a7fa-7a2ace86d608"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:28.034599642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 518 len 24576 data: 18 15 16 17 15 15 03/30 Read block 502 len 32768 04/30 Read block 16 len 32768 05/30 Flush 06/30 Flush 07/30 Write block 124 len 20480 data: 10 10 9 10 8 08/30 Read block 551 len 36864 09/30 Write block 566 len 40960 data: 15 15 18 16 17 22 20 20 19 21 10/30 Write block 584 len 12288 data: 10 11 12 11/30 Write block 469 len 20480 data: 11 11 10 8 11 12/30 Read block 325 len 16384 13/30 Flush 14/30 Write block 126 len 4096 data: 10 15/30 Read block 413 len 16384 16/30 Read block 241 len 40960 17/30 Write block 332 len 16384 data: 14 16 14 11 18/30 Read block 347 len 28672 19/30 Write block 201 len 36864 data: 11 14 14 14 20 20 22 19 16 20/30 Write block 160 len 32768 data: 19 17 20 21 20 23 21 22 21/30 Read block 583 len 16384 22/30 Write block 297 len 24576 data: 15 14 17 16 14 14 23/30 Write block 240 len 24576 data: 13 11 13 13 13 11 24/30 Write block 572 len 32768 data: 21 21 20 22 19 15 16 13 25/30 Write block 500 len 12288 data: 16 17 14 26/30 Write block 447 len 8192 data: 9 10 27/30 Write block 252 len 40960 data: 17 19 20 14 14 16 20 18 15 15 28/30 Write block 213 len 32768 data: 14 11 13 13 13 11 13 13 29/30 Read block 84 len 24576 30/30 Read block 570 len 16384 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 24 Acked 1023 Write 6 Done Done Done false 25 Acked 1024 Write 8 Done Done Done false 26 Acked 1025 Write 3 Done Done Done false 27 Acked 1026 Write 2 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 8 Done Done Done false 30 Acked 1029 Read 6 Done Done Done false 31 Acked 1030 Read 4 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 1 1 Done 8 8 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 31 30 29 28 23 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:28.06166018Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1895} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 175 175 175 358 358 358 F F F 6 120-139 178 178 178 368 368 368 F F F 7 140-159 176 176 176 362 362 362 F F F 8 160-179 178 178 178 368 368 368 F F F 9 180-199 176 176 176 361 361 361 F F F 10 200-219 178 178 178 368 368 368 T T T 11 220-239 173 173 173 354 354 354 T T T 12 240-259 177 177 177 363 363 363 T T T 13 260-279 177 177 177 363 363 363 T T T 14 280-299 178 178 178 368 368 368 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 178 178 178 368 368 368 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 174 174 174 355 355 355 F F F 19 380-399 175 175 175 358 358 358 F F F 20 400-419 177 177 177 363 363 363 F F F 21 420-439 177 177 177 363 363 363 F F F 22 440-459 176 176 176 362 362 362 T T T 23 460-479 178 178 178 367 367 367 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 178 178 178 365 365 365 T T T 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 178 178 178 367 367 367 T T T 29 580-599 178 178 178 367 367 367 F F F Max gen: 178, Max flush: 368 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 175 175 175 358 358 358 F F F 6 120-139 178 178 178 368 368 368 F F F 7 140-159 176 176 176 362 362 362 F F F 8 160-179 178 178 178 368 368 368 F F F 9 180-199 176 176 176 361 361 361 F F F 10 200-219 179 179 179 369 369 369 F F F 11 220-239 179 179 179 369 369 369 F F F 12 240-259 179 179 179 369 369 369 F F F 13 260-279 179 179 179 369 369 369 F F F 14 280-299 178 178 178 368 368 368 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 178 178 178 368 368 368 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 174 174 174 355 355 355 F F F 19 380-399 175 175 175 358 358 358 F F F 20 400-419 177 177 177 363 363 363 F F F 21 420-439 177 177 177 363 363 363 F F F 22 440-459 179 179 179 369 369 369 F F F 23 460-479 178 178 178 367 367 367 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 179 179 179 369 369 369 F F F 29 580-599 178 178 178 367 367 367 F F F Max gen: 179, Max flush: 369 Begin loop 90 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.304206603Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.304594276Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.304617162Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903} {"msg":"Crucible d631636a-f54a-421b-b1c1-7e628367f012 has session id: 5b2f9768-3e3e-422b-842b-1bee52c2a315","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.304861816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Upstairs opts: Upstairs UUID: d631636a-f54a-421b-b1c1-7e628367f012, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.30496585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Crucible stats registered with UUID: d631636a-f54a-421b-b1c1-7e628367f012","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.304993976Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305347904Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903} {"msg":"d631636a-f54a-421b-b1c1-7e628367f012 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305409415Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.30544124Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305468956Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305497592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305522188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305547164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305571051Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305593857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305616584Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305639481Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:42.305684844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:42.305716939Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:42.305747135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305769931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305794318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_connection connected from Ok(127.0.0.1:44642)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305819054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.305852779Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_connection connected from Ok(127.0.0.1:62181)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.306036512Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.30665354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.306681046Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.306696094Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.306709862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307234354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307339109Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307785523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307807799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307822387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307907425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.307925592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:42.3079388Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.307186241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_connection connected from Ok(127.0.0.1:54682)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.307236614Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"io task","client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.30827729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.308318284Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.30888135Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309430619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309451816Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[0]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 368, 362, 368, 361, 369, 369]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309467153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[0]R generation[0..12]: [175, 174, 173, 173, 173, 175, 178, 176, 178, 176, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309481471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309495529Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[1]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 368, 362, 368, 361, 369, 369]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309509597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[1]R generation[0..12]: [175, 174, 173, 173, 173, 175, 178, 176, 178, 176, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309523335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309537263Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[2]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 358, 368, 362, 368, 361, 369, 369]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309550851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[2]R generation[0..12]: [175, 174, 173, 173, 173, 175, 178, 176, 178, 176, 179, 179]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309564469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309578257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309594784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309608382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Next flush: 370","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.30962498Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309637998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"":"downstairs","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309651346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309663184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"0","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309676632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"1","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.30968979Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"client":"2","":"downstairs","session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309702808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309714817Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"d631636a-f54a-421b-b1c1-7e628367f012 is now active with session: 5b2f9768-3e3e-422b-842b-1bee52c2a315","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309727455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309739253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903,"session_id":"5b2f9768-3e3e-422b-842b-1bee52c2a315"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:44:52.309751171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 294 len 24576 data: 14 13 13 16 15 18 02/30 Read block 536 len 8192 03/30 Write block 105 len 32768 data: 11 13 13 13 18 18 21 22 04/30 Read block 406 len 20480 05/30 Flush 06/30 Flush 07/30 Write block 371 len 8192 data: 13 9 08/30 Flush 09/30 Write block 360 len 12288 data: 11 12 10 10/30 Write block 562 len 8192 data: 15 16 11/30 Flush 12/30 Write block 148 len 4096 data: 10 13/30 Read block 105 len 8192 14/30 Read block 259 len 16384 15/30 Flush 16/30 Write block 404 len 36864 data: 18 21 21 17 15 16 14 15 14 17/30 Write block 579 len 20480 data: 14 17 18 15 10 18/30 Write block 582 len 12288 data: 16 11 11 19/30 Write block 227 len 20480 data: 6 7 8 7 7 20/30 Read block 286 len 28672 21/30 Read block 463 len 20480 22/30 Read block 166 len 28672 23/30 Write block 150 len 12288 data: 11 12 10 24/30 Read block 311 len 16384 25/30 Read block 525 len 12288 26/30 Write block 383 len 24576 data: 13 12 11 12 10 14 27/30 Read block 165 len 12288 28/30 Read block 256 len 4096 29/30 Read block 120 len 12288 30/30 Read block 579 len 8192 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 16 Acked 1015 Write 9 Done Done Done false 17 Acked 1016 Write 5 Done Done Done false 18 Acked 1017 Write 3 Done Done Done false 19 Acked 1018 Write 5 Done Done Done false 20 Acked 1019 Read 7 Done Done Done false 21 Acked 1020 Read 5 Done Done Done false 22 Acked 1021 Read 7 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 4 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 6 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 1 Done Done Done false 29 Acked 1028 Read 3 Done Done Done false 30 Acked 1029 Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:44:52.335455431Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1903} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 180 180 180 370 370 370 F F F 6 120-139 178 178 178 368 368 368 F F F 7 140-159 180 180 180 374 374 374 T T T 8 160-179 178 178 178 368 368 368 F F F 9 180-199 176 176 176 361 361 361 F F F 10 200-219 179 179 179 369 369 369 F F F 11 220-239 179 179 179 369 369 369 T T T 12 240-259 179 179 179 369 369 369 F F F 13 260-279 179 179 179 369 369 369 F F F 14 280-299 180 180 180 370 370 370 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 178 178 178 368 368 368 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 180 180 180 373 373 373 F F F 19 380-399 175 175 175 358 358 358 T T T 20 400-419 177 177 177 363 363 363 T T T 21 420-439 177 177 177 363 363 363 F F F 22 440-459 179 179 179 369 369 369 F F F 23 460-479 178 178 178 367 367 367 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 180 180 180 373 373 373 T T T 29 580-599 178 178 178 367 367 367 T T T Max gen: 180, Max flush: 374 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 358 358 358 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 173 173 173 354 354 354 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 180 180 180 370 370 370 F F F 6 120-139 178 178 178 368 368 368 F F F 7 140-159 181 181 181 375 375 375 F F F 8 160-179 178 178 178 368 368 368 F F F 9 180-199 176 176 176 361 361 361 F F F 10 200-219 179 179 179 369 369 369 F F F 11 220-239 181 181 181 375 375 375 F F F 12 240-259 179 179 179 369 369 369 F F F 13 260-279 179 179 179 369 369 369 F F F 14 280-299 180 180 180 370 370 370 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 178 178 178 368 368 368 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 180 180 180 373 373 373 F F F 19 380-399 181 181 181 375 375 375 F F F 20 400-419 181 181 181 375 375 375 F F F 21 420-439 177 177 177 363 363 363 F F F 22 440-459 179 179 179 369 369 369 F F F 23 460-479 178 178 178 367 367 367 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 181 181 181 375 375 375 F F F 29 580-599 181 181 181 375 375 375 F F F Max gen: 181, Max flush: 375 Begin loop 91 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.622236259Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.622612133Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.622639119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911} {"msg":"Crucible 5e34663b-e59c-4117-8544-739f3ad83078 has session id: 9a96b91e-1526-428c-8885-bf87e3af1011","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.622778429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Upstairs opts: Upstairs UUID: 5e34663b-e59c-4117-8544-739f3ad83078, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.622870405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Crucible stats registered with UUID: 5e34663b-e59c-4117-8544-739f3ad83078","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.62290095Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623188988Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911} {"msg":"5e34663b-e59c-4117-8544-739f3ad83078 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623264057Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623285323Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623300641Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623314309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623327497Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623340525Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623361482Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623385508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623410435Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623471826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_connection connected from Ok(127.0.0.1:55776)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623566672Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_connection connected from Ok(127.0.0.1:48667)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623596457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:06.623929048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:06.623958063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:06.623974621Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.623991279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.624004967Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.624105782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.62445642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.624497104Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.624513411Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.624533188Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625031575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625163225Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625524971Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625554257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625570145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625582973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625596331Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:06.625609659Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.625756367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_connection connected from Ok(127.0.0.1:58863)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.625985853Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"io task","client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.626863273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.626891539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.627477222Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628025871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628053756Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[0]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 370, 368, 375, 368, 361, 369, 375]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628070294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[0]R generation[0..12]: [175, 174, 173, 173, 173, 180, 178, 181, 178, 176, 179, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628085022Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.62809941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[1]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 370, 368, 375, 368, 361, 369, 375]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628113618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[1]R generation[0..12]: [175, 174, 173, 173, 173, 180, 178, 181, 178, 176, 179, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628127445Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628141503Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[2]R flush_numbers[0..12]: [358, 356, 354, 354, 354, 370, 368, 375, 368, 361, 369, 375]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628155241Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"[2]R generation[0..12]: [175, 174, 173, 173, 173, 180, 178, 181, 178, 176, 179, 181]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628168839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628189626Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628203844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628217352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Next flush: 376","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.6282302Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628242959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"":"downstairs","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628256377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628268425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"0","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628281763Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"1","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628294871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"client":"2","":"downstairs","session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628308199Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628320517Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"5e34663b-e59c-4117-8544-739f3ad83078 is now active with session: 9a96b91e-1526-428c-8885-bf87e3af1011","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628333055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628344923Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911,"session_id":"9a96b91e-1526-428c-8885-bf87e3af1011"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:16.628356922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 20480 data: 7 8 8 9 11 02/30 Write block 5 len 36864 data: 15 15 15 14 17 19 19 16 11 03/30 Read block 520 len 8192 04/30 Write block 383 len 4096 data: 14 05/30 Read block 494 len 40960 06/30 Write block 453 len 24576 data: 10 9 9 8 8 7 07/30 Read block 444 len 12288 08/30 Write block 229 len 20480 data: 9 8 8 10 10 09/30 Write block 585 len 4096 data: 12 10/30 Write block 69 len 4096 data: 19 11/30 Read block 367 len 16384 12/30 Write block 365 len 8192 data: 14 17 13/30 Read block 580 len 20480 14/30 Read block 303 len 16384 15/30 Write block 63 len 40960 data: 16 17 16 15 16 15 20 12 10 10 16/30 Read block 63 len 20480 17/30 Read block 455 len 12288 18/30 Write block 199 len 8192 data: 12 13 19/30 Write block 111 len 40960 data: 22 23 25 20 18 17 12 14 13 12 20/30 Read block 416 len 20480 21/30 Read block 274 len 36864 22/30 Read block 307 len 24576 23/30 Write block 465 len 12288 data: 12 12 16 24/30 Read block 364 len 12288 25/30 Read block 146 len 12288 26/30 Read block 312 len 20480 27/30 Write block 407 len 28672 data: 18 16 17 15 16 15 13 28/30 Write block 279 len 32768 data: 14 14 13 12 15 16 16 18 29/30 Write block 564 len 16384 data: 13 12 16 16 30/30 Write block 323 len 32768 data: 13 15 16 17 15 16 14 13 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:1 downstairs:31 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1 Acked 1000 Write 5 Done Done Done false 2 Acked 1001 Write 9 Done Done Done false 3 Acked 1002 Read 2 Done Done Done false 4 Acked 1003 Write 1 Done Done Done false 5 Acked 1004 Read 10 Done Done Done false 6 Acked 1005 Write 6 Done Done Done false 7 Acked 1006 Read 3 Done Done Done false 8 Acked 1007 Write 5 Done Done Done false 9 Acked 1008 Write 1 Done Done Done false 10 Acked 1009 Write 1 Done Done Done false 11 Acked 1010 Read 4 Done Done Done false 12 Acked 1011 Write 2 Done Done Done false 13 Acked 1012 Read 5 Done Done Done false 14 Acked 1013 Read 4 Done Done Done false 15 Acked 1014 Write 10 Done Done Done false 16 Acked 1015 Read 5 Done Done Done false 17 Acked 1016 Read 3 Done Done Done false 18 Acked 1017 Write 2 Done Done Done false 19 Acked 1018 Write 10 Done Done Done false 20 Acked 1019 Read 5 Done Done Done false 21 Acked 1020 Read 9 Done Done Done false 22 Acked 1021 Read 6 Done Done Done false 23 Acked 1022 Write 3 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Read 5 Done Done Done false 27 NotAcked 1026 Flush 0 Sent Sent Sent false 28 Acked 1027 Write 7 Sent Sent Sent false 29 Acked 1028 Write 8 Sent Sent Sent false 30 Acked 1029 Write 4 Sent Sent Sent false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 5 5 5 15 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 31 30 29 28 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:16.649703207Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1911} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 376 376 376 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 182 182 182 376 376 376 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 182 182 182 376 376 376 F F F 6 120-139 182 182 182 376 376 376 F F F 7 140-159 181 181 181 375 375 375 F F F 8 160-179 178 178 178 368 368 368 F F F 9 180-199 182 182 182 376 376 376 F F F 10 200-219 182 182 182 376 376 376 F F F 11 220-239 182 182 182 376 376 376 F F F 12 240-259 179 179 179 369 369 369 F F F 13 260-279 179 179 179 369 369 369 T T T 14 280-299 180 180 180 370 370 370 T T T 15 300-319 178 178 178 368 368 368 F F F 16 320-339 178 178 178 368 368 368 F F T <--- 17 340-359 176 176 176 361 361 361 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 182 182 182 376 376 376 F F F 20 400-419 181 181 181 375 375 375 T T T 21 420-439 177 177 177 363 363 363 F F F 22 440-459 182 182 182 376 376 376 F F F 23 460-479 182 182 182 376 376 376 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 181 181 181 375 375 375 F T T <--- 29 580-599 182 182 182 376 376 376 F F F Max gen: 182, Max flush: 376 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 376 376 376 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 182 182 182 376 376 376 F F F 4 080-099 173 173 173 354 354 354 F F F 5 100-119 182 182 182 376 376 376 F F F 6 120-139 182 182 182 376 376 376 F F F 7 140-159 181 181 181 375 375 375 F F F 8 160-179 178 178 178 368 368 368 F F F 9 180-199 182 182 182 376 376 376 F F F 10 200-219 182 182 182 376 376 376 F F F 11 220-239 182 182 182 376 376 376 F F F 12 240-259 179 179 179 369 369 369 F F F 13 260-279 183 183 183 377 377 377 F F F 14 280-299 183 183 183 377 377 377 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 183 183 183 377 377 377 F F F 17 340-359 176 176 176 361 361 361 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 182 182 182 376 376 376 F F F 20 400-419 183 183 183 377 377 377 F F F 21 420-439 177 177 177 363 363 363 F F F 22 440-459 182 182 182 376 376 376 F F F 23 460-479 182 182 182 376 376 376 F F F 24 480-499 177 177 177 363 363 363 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 183 183 183 377 377 377 F F F 29 580-599 182 182 182 376 376 376 F F F Max gen: 183, Max flush: 377 Begin loop 92 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.869163355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.869474289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.869496336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919} {"msg":"Crucible edae3edd-9b45-4966-a0d1-b5ac411f7829 has session id: 2ace6e1e-1e0a-4499-ba06-091db2606958","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.8699377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Upstairs opts: Upstairs UUID: edae3edd-9b45-4966-a0d1-b5ac411f7829, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.869986163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Crucible stats registered with UUID: edae3edd-9b45-4966-a0d1-b5ac411f7829","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870014329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870306955Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919} {"msg":"edae3edd-9b45-4966-a0d1-b5ac411f7829 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.87034446Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870359457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870374455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870397412Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870423048Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870447355Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870469651Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870492898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870519034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.87054425Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_connection connected from Ok(127.0.0.1:39012)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.870842906Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_connection connected from Ok(127.0.0.1:39142)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871144541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:30.871191554Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:30.87122062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:30.871250456Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871276072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871301868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871328164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871569278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871606943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871778457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.871816592Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872088221Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872206884Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872632311Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872667106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872690392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872710289Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872732356Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:30.872753863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.872575229Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_connection connected from Ok(127.0.0.1:64082)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.872626261Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"io task","client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.873668557Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.873696462Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.874335448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.874971713Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.8749983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[0]R flush_numbers[0..12]: [376, 356, 354, 376, 354, 376, 376, 375, 368, 376, 376, 376]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875015027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[0]R generation[0..12]: [182, 174, 173, 182, 173, 182, 182, 181, 178, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875030085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875049582Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[1]R flush_numbers[0..12]: [376, 356, 354, 376, 354, 376, 376, 375, 368, 376, 376, 376]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.87506392Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[1]R generation[0..12]: [182, 174, 173, 182, 173, 182, 182, 181, 178, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875077788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875091746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[2]R flush_numbers[0..12]: [376, 356, 354, 376, 354, 376, 376, 375, 368, 376, 376, 376]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875105344Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[2]R generation[0..12]: [182, 174, 173, 182, 173, 182, 182, 181, 178, 182, 182, 182]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875118982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.8751328Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":Wait for a query_work_queue command to finish before sending IO "2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875149147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875162235Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Next flush: 378","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875175003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875187681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"":"downstairs","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.87520085Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875212668Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"0","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875226216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"1","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875239574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"client":"2","":"downstairs","session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875254931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.87526723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"edae3edd-9b45-4966-a0d1-b5ac411f7829 is now active with session: 2ace6e1e-1e0a-4499-ba06-091db2606958","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875279798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875291746Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919,"session_id":"2ace6e1e-1e0a-4499-ba06-091db2606958"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:40.875303774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 427 len 16384 data: 15 13 15 16 02/30 Write block 186 len 40960 data: 9 10 10 11 10 11 12 10 10 13 03/30 Write block 229 len 8192 data: 10 9 04/30 Write block 80 len 24576 data: 13 15 15 16 15 11 05/30 Write block 173 len 28672 data: 14 14 16 11 12 10 9 06/30 Write block 66 len 12288 data: 16 17 16 07/30 Write block 334 len 28672 data: 15 12 13 14 13 14 13 08/30 Write block 140 len 4096 data: 13 09/30 Read block 252 len 40960 10/30 Read block 272 len 12288 11/30 Write block 472 len 40960 data: 9 12 11 13 16 16 18 18 20 17 12/30 Read block 552 len 12288 13/30 Write block 65 len 4096 data: 17 14/30 Flush 15/30 Write block 249 len 16384 data: 13 15 16 18 16/30 Read block 120 len 12288 17/30 Read block 379 len 40960 18/30 Read block 541 len 24576 19/30 Read block 62 len 20480 20/30 Read block 257 len 20480 21/30 Read block 18 len 8192 22/30 Read block 297 len 16384 23/30 Write block 192 len 4096 data: 13 24/30 Write block 443 len 8192 data: 17 13 25/30 Read block 528 len 4096 26/30 Flush 27/30 Write block 434 len 8192 data: 17 15 28/30 Read block 115 len 40960 29/30 Write block 466 len 28672 data: 13 17 14 12 12 11 10 30/30 Read block 217 len 4096 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Read 10 Done Done Done false 30 Acked 1029 Write 7 Done Sent Done false 31 Acked 1030 Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 0 2 0 2 Done 4 2 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:40.901812845Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1919} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 376 376 376 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 184 184 184 378 378 378 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 182 182 182 376 376 376 F F F 6 120-139 182 182 182 376 376 376 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 184 184 184 380 380 380 F F F 10 200-219 182 182 182 376 376 376 F F F 11 220-239 184 184 184 378 378 378 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 183 183 183 377 377 377 F F F 14 280-299 183 183 183 377 377 377 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 184 184 184 378 378 378 F F F 17 340-359 184 184 184 378 378 378 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 182 182 182 376 376 376 F F F 20 400-419 183 183 183 377 377 377 F F F 21 420-439 184 184 184 378 378 378 T T T 22 440-459 184 184 184 380 380 380 F F F 23 460-479 184 184 184 378 378 378 T T T 24 480-499 184 184 184 378 378 378 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 183 183 183 377 377 377 F F F 29 580-599 182 182 182 376 376 376 F F F Max gen: 184, Max flush: 380 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 376 376 376 F F F 1 020-039 174 174 174 356 356 356 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 184 184 184 378 378 378 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 182 182 182 376 376 376 F F F 6 120-139 182 182 182 376 376 376 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 184 184 184 380 380 380 F F F 10 200-219 182 182 182 376 376 376 F F F 11 220-239 184 184 184 378 378 378 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 183 183 183 377 377 377 F F F 14 280-299 183 183 183 377 377 377 F F F 15 300-319 178 178 178 368 368 368 F F F 16 320-339 184 184 184 378 378 378 F F F 17 340-359 184 184 184 378 378 378 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 182 182 182 376 376 376 F F F 20 400-419 183 183 183 377 377 377 F F F 21 420-439 185 185 185 381 381 381 F F F 22 440-459 184 184 184 380 380 380 F F F 23 460-479 185 185 185 381 381 381 F F F 24 480-499 184 184 184 378 378 378 F F F 25 500-519 179 179 179 369 369 369 F F F 26 520-539 178 178 178 365 365 365 F F F 27 540-559 167 167 167 340 340 340 F F F 28 560-579 183 183 183 377 377 377 F F F 29 580-599 182 182 182 376 376 376 F F F Max gen: 185, Max flush: 381 Begin loop 93 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.084079803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.084620363Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.084655038Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927} {"msg":"Crucible 9179699b-6351-41e1-8857-3293c853593c has session id: ba6f4702-26f9-4902-bddd-6d895e25e164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.084721898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Upstairs opts: Upstairs UUID: 9179699b-6351-41e1-8857-3293c853593c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.084770291Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Crucible stats registered with UUID: 9179699b-6351-41e1-8857-3293c853593c","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.084796787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085092113Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927} {"msg":"9179699b-6351-41e1-8857-3293c853593c active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085121569Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085142826Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085165912Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085187629Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085209386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085230753Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085252599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085273986Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085295533Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.08531799Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_connection connected from Ok(127.0.0.1:58394)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085572382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_connection connected from Ok(127.0.0.1:39259)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085610217Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:55.085782871Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:55.085824745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:45:55.085860929Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085885106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085910912Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.085972803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.086497985Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.086524601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.086539469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.086552897Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087103235Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087227147Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087573295Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087600111Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087614589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087736851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087755808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:45:55.087769206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.086947439Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_connection connected from Ok(127.0.0.1:34107)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.087178874Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"io task","client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.088169087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.088197303Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.088768039Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089322866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089348683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[0]R flush_numbers[0..12]: [376, 356, 354, 378, 378, 376, 376, 378, 378, 380, 376, 378]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.08936458Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[0]R generation[0..12]: [182, 174, 173, 184, 184, 182, 182, 184, 184, 184, 182, 184]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089379448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089398405Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[1]R flush_numbers[0..12]: [376, 356, 354, 378, 378, 376, 376, 378, 378, 380, 376, 378]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089412863Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[1]R generation[0..12]: [182, 174, 173, 184, 184, 182, 182, 184, 184, 184, 182, 184]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089426801Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089440979Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[2]R flush_numbers[0..12]: [376, 356, 354, 378, 378, 376, 376, 378, 378, 380, 376, 378]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089454777Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[2]R generation[0..12]: [182, 174, 173, 184, 184, 182, 182, 184, 184, 184, 182, 184]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089468335Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089482313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:46:05.089494961Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089508279Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Next flush: 382","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089521017Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089533875Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"":"downstairs","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089547153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089558981Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"0","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089572219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"1","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089585717Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"client":"2","":"downstairs","session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089601165Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089613523Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"9179699b-6351-41e1-8857-3293c853593c is now active with session: ba6f4702-26f9-4902-bddd-6d895e25e164","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089626271Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.08963823Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:05.089650418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 416 len 20480 data: 11 9 7 7 10 02/30 Write block 288 len 36864 data: 19 21 21 21 18 14 15 14 14 03/30 Write block 492 len 36864 data: 16 15 18 16 18 21 19 17 17 04/30 Write block 280 len 4096 data: 15 05/30 Read block 192 len 36864 06/30 Read block 551 len 20480 07/30 Read block 502 len 16384 08/30 Read block 212 len 20480 09/30 Write block 319 len 28672 data: 10 10 9 10 14 16 17 10/30 Read block 232 len 32768 11/30 Read block 350 len 36864 12/30 Read block 224 len 32768 13/30 Flush 14/30 Write block 572 len 20480 data: 22 22 21 23 20 15/30 Read block 76 len 8192 16/30 Read block 67 len 16384 17/30 Flush 18/30 Write block 211 len 20480 data: 19 16 15 12 14 19/30 Read block 170 len 28672 20/30 Read block 94 len 8192 21/30 Flush 22/30 Write block 340 len 20480 data: 14 13 13 13 13 23/30 Write block 6 len 8192 data: 16 16 24/30 Write block 551 len 20480 data: 7 5 7 8 10 25/30 Read block 50 len 12288 26/30 Write block 187 len 16384 data: 11 11 12 11 27/30 Write block 15 len 28672 data: 11 10 10 8 9 10 9 28/30 Read block 541 len 20480 29/30 Write block 526 len 4096 data: 11 30/30 Write block 264 len 32768 data: 12 16 10 11 11 13 12 14 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 5 Done Done Done false 24 Acked 1023 Write 2 Done Done Done false 25 Acked 1024 Write 5 Done Done Done false 26 Acked 1025 Read 3 Done Done Done false 27 Acked 1026 Write 4 Done Done Done false 28 Acked 1027 Write 7 Done Done Done false 29 Acked 1028 Read 5 Done Done Done false 30 Acked 1029 Write 1 Done Done Done false 31 Acked 1030 Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117238618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117690152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117726866Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117751122Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11776959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117787077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117805585Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117819052Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117832921Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117853917Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117874784Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117895381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117916628Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117938545Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117959842Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.117980808Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118003025Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118032541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118061296Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118085353Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118100071Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118118258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118135146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.1181745Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118189437Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118202266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118214314Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118226242Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11823831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118250408Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118262647Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118274875Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118287153Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118299411Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11831163Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118326417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118338586Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118350674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118362702Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11837456Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118386708Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118398757Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118410695Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118422663Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118434711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118446599Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118458418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118470266Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118482294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118494152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118506061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118517969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118537466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118556803Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118598537Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118611495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118651469Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118675296Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118687504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118699652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11871161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118723818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118735807Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118747665Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118759733Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118771581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118783329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118795058Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118807216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118822244Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118834452Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11884646Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118858278Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118870467Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118882615Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118894843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118906851Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11891898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118931108Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118943206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118955204Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118967072Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11897898Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.118990769Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119002527Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119014345Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119028033Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119040151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11905206Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119064798Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119076666Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119088574Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119100382Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119112351Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119124409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119136377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119148475Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119160334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119172272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.1191843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119196438Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119208286Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119220325Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119234093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119246201Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119258129Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119270097Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119281946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119294054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119305992Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11931788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119329848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119341577Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119353455Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119365463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119377441Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.11938951Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119401498Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119413556Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119425424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119438912Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119450771Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119462739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119474457Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119486365Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:05.119498354Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1927,"session_id":"ba6f4702-26f9-4902-bddd-6d895e25e164"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 182 182 182 376 376 376 T T T 1 020-039 174 174 174 356 356 356 T T T 2 040-059 173 173 173 354 354 354 F F F 3 060-079 184 184 184 378 378 378 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 182 182 182 376 376 376 F F F 6 120-139 182 182 182 376 376 376 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 184 184 184 380 380 380 T T T 10 200-219 186 186 186 385 385 385 F F F 11 220-239 184 184 184 378 378 378 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 183 183 183 377 377 377 T T T 14 280-299 186 186 186 382 382 382 F F F 15 300-319 186 186 186 382 382 382 F F F 16 320-339 186 186 186 382 382 382 F F F 17 340-359 184 184 184 378 378 378 T T T 18 360-379 182 182 182 376 376 376 F F F 19 380-399 182 182 182 376 376 376 F F F 20 400-419 186 186 186 382 382 382 F F F 21 420-439 186 186 186 382 382 382 F F F 22 440-459 184 184 184 380 380 380 F F F 23 460-479 185 185 185 381 381 381 F F F 24 480-499 186 186 186 382 382 382 F F F 25 500-519 186 186 186 382 382 382 F F F 26 520-539 178 178 178 365 365 365 T T T 27 540-559 167 167 167 340 340 340 T T T 28 560-579 186 186 186 383 383 383 F F F 29 580-599 182 182 182 376 376 376 F F F Max gen: 186, Max flush: 385 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 386 386 386 F F F 1 020-039 187 187 187 386 386 386 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 184 184 184 378 378 378 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 182 182 182 376 376 376 F F F 6 120-139 182 182 182 376 376 376 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 187 187 187 386 386 386 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 184 184 184 378 378 378 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 187 187 187 386 386 386 F F F 14 280-299 186 186 186 382 382 382 F F F 15 300-319 186 186 186 382 382 382 F F F 16 320-339 186 186 186 382 382 382 F F F 17 340-359 187 187 187 386 386 386 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 182 182 182 376 376 376 F F F 20 400-419 186 186 186 382 382 382 F F F 21 420-439 186 186 186 382 382 382 F F F 22 440-459 184 184 184 380 380 380 F F F 23 460-479 185 185 185 381 381 381 F F F 24 480-499 186 186 186 382 382 382 F F F 25 500-519 186 186 186 382 382 382 F F F 26 520-539 187 187 187 386 386 386 F F F 27 540-559 187 187 187 386 386 386 F F F 28 560-579 186 186 186 383 383 383 F F F 29 580-599 182 182 182 376 376 376 F F F Max gen: 187, Max flush: 386 Begin loop 94 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.375653821Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376036614Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376058031Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935} {"msg":"Crucible e8020966-e359-47db-b14d-b1d455121851 has session id: 3e507e25-4aa0-4d32-ba93-cf6bca586f60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376378034Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Upstairs opts: Upstairs UUID: e8020966-e359-47db-b14d-b1d455121851, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376416588Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Crucible stats registered with UUID: e8020966-e359-47db-b14d-b1d455121851","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376433205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.3767378Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935} {"msg":"e8020966-e359-47db-b14d-b1d455121851 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376771505Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376795381Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376821858Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376845364Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.37687285Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376897237Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376924402Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376949749Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.376977954Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377028717Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:19.377168336Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:19.37727483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:19.377306186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377329062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377354879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_connection connected from Ok(127.0.0.1:49155)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377381905Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377408581Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_connection connected from Ok(127.0.0.1:33444)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377435967Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.377845106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.37788403Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378010212Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378037448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378374667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378484571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378871844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.37889872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378913847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.378987587Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.379006724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:19.379020332Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.378604193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_connection connected from Ok(127.0.0.1:56591)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.378723196Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"io task","client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.379811704Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.379846309Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380342026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380845501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380868857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[0]R flush_numbers[0..12]: [386, 386, 354, 378, 378, 376, 376, 378, 378, 386, 385, 378]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380884555Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[0]R generation[0..12]: [187, 187, 173, 184, 184, 182, 182, 184, 184, 187, 186, 184]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380899843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380914371Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[1]R flush_numbers[0..12]: [386, 386, 354, 378, 378, 376, 376, 378, 378, 386, 385, 378]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380928139Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[1]R generation[0..12]: [187, 187, 173, 184, 184, 182, 182, 184, 184, 187, 186, 184]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380941857Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380956075Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[2]R flush_numbers[0..12]: [386, 386, 354, 378, 378, 376, 376, 378, 378, 386, 385, 378]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380969843Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[2]R generation[0..12]: [187, 187, 173, 184, 184, 182, 182, 184, 184, 187, 186, 184]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380983501Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.380997598Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381010487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381023735Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairsWait for a query_work_queue command to finish before sending IO ","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Next flush: 387","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381044042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.38105705Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"":"downstairs","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381070418Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381082506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"0","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381095804Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"1","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381108782Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"client":"2","":"downstairs","session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.38112193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381133948Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"e8020966-e359-47db-b14d-b1d455121851 is now active with session: 3e507e25-4aa0-4d32-ba93-cf6bca586f60","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381146936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381158765Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935,"session_id":"3e507e25-4aa0-4d32-ba93-cf6bca586f60"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:29.381170953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 393 len 40960 data: 16 17 18 20 20 17 15 18 18 19 02/30 Read block 580 len 24576 03/30 Write block 221 len 24576 data: 12 11 9 6 7 6 04/30 Write block 486 len 24576 data: 24 20 23 23 21 17 05/30 Read block 307 len 20480 06/30 Write block 556 len 28672 data: 14 11 11 10 11 15 16 07/30 Write block 381 len 20480 data: 16 16 15 13 12 08/30 Write block 463 len 28672 data: 9 10 13 14 18 15 13 09/30 Read block 338 len 20480 10/30 Write block 322 len 24576 data: 11 15 17 18 18 16 11/30 Write block 516 len 24576 data: 20 18 19 16 17 18 12/30 Write block 592 len 24576 data: 9 10 10 12 11 10 13/30 Read block 312 len 32768 14/30 Read block 464 len 40960 15/30 Write block 223 len 32768 data: 10 7 8 7 7 8 11 10 16/30 Write block 458 len 40960 data: 8 7 7 8 9 10 11 14 15 19 17/30 Write block 116 len 20480 data: 18 13 15 14 13 18/30 Read block 144 len 24576 19/30 Read block 74 len 4096 20/30 Read block 17 len 28672 21/30 Read block 125 len 28672 22/30 Read block 333 len 16384 23/30 Read block 25 len 20480 24/30 Write block 293 len 36864 data: 15 16 15 15 17 16 19 17 15 25/30 Read block 382 len 40960 26/30 Read block 541 len 12288 27/30 Read block 147 len 32768 28/30 Read block 455 len 32768 29/30 Read block 296 len 32768 30/30 Write block 12 len 24576 data: 17 12 10 12 11 11 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Read 4 Done Done Done false 24 Acked 1023 Read 5 Done Done Done false 25 Acked 1024 Write 9 Done Done Done false 26 Acked 1025 Read 10 Done Done Done false 27 Acked 1026 Read 3 Done Done Done false 28 Acked 1027 Read 8 Done Done Done false 29 Acked 1028 Read 8 Done Done Done false 30 Acked 1029 Read 8 Done Done Done false 31 Acked 1030 Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:29.40918539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1935} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 187 187 187 386 386 386 T T T 1 020-039 187 187 187 386 386 386 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 184 184 184 378 378 378 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 188 188 188 387 387 387 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 187 187 187 386 386 386 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 188 188 188 387 387 387 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 187 187 187 386 386 386 F F F 14 280-299 186 186 186 382 382 382 T T T 15 300-319 186 186 186 382 382 382 T T T 16 320-339 188 188 188 387 387 387 F F F 17 340-359 187 187 187 386 386 386 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 188 188 188 387 387 387 F F F 20 400-419 188 188 188 387 387 387 F F F 21 420-439 186 186 186 382 382 382 F F F 22 440-459 188 188 188 387 387 387 F F F 23 460-479 188 188 188 387 387 387 F F F 24 480-499 188 188 188 387 387 387 F F F 25 500-519 188 188 188 387 387 387 F F F 26 520-539 188 188 188 387 387 387 F F F 27 540-559 188 188 188 387 387 387 F F F 28 560-579 188 188 188 387 387 387 F F F 29 580-599 188 188 188 387 387 387 F F F Max gen: 188, Max flush: 387 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 388 388 388 F F F 1 020-039 187 187 187 386 386 386 F F F 2 040-059 173 173 173 354 354 354 F F F 3 060-079 184 184 184 378 378 378 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 188 188 188 387 387 387 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 187 187 187 386 386 386 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 188 188 188 387 387 387 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 187 187 187 386 386 386 F F F 14 280-299 189 189 189 388 388 388 F F F 15 300-319 189 189 189 388 388 388 F F F 16 320-339 188 188 188 387 387 387 F F F 17 340-359 187 187 187 386 386 386 F F F 18 360-379 182 182 182 376 376 376 F F F 19 380-399 188 188 188 387 387 387 F F F 20 400-419 188 188 188 387 387 387 F F F 21 420-439 186 186 186 382 382 382 F F F 22 440-459 188 188 188 387 387 387 F F F 23 460-479 188 188 188 387 387 387 F F F 24 480-499 188 188 188 387 387 387 F F F 25 500-519 188 188 188 387 387 387 F F F 26 520-539 188 188 188 387 387 387 F F F 27 540-559 188 188 188 387 387 387 F F F 28 560-579 188 188 188 387 387 387 F F F 29 580-599 188 188 188 387 387 387 F F F Max gen: 189, Max flush: 388 Begin loop 95 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.607974541Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.608241101Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.608261978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943} {"msg":"Crucible 6ca606cd-1215-47b4-ae92-e623b8869895 has session id: 044c1c73-2001-4aba-b3c2-eccf9f6dd101","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.608612306Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Upstairs opts: Upstairs UUID: 6ca606cd-1215-47b4-ae92-e623b8869895, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.60865558Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Crucible stats registered with UUID: 6ca606cd-1215-47b4-ae92-e623b8869895","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.608679186Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.60899294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943} {"msg":"6ca606cd-1215-47b4-ae92-e623b8869895 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609042583Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.60905883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609077398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609091485Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609104773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609117681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609140078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609165274Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609189791Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609211787Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_connection connected from Ok(127.0.0.1:50458)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609409818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_connection connected from Ok(127.0.0.1:58656)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609449822Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:43.609652572Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:43.609692346Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:43.609719352Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609739359Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609761136Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.609791821Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.610262062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.610288508Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.610391313Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.610418819Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.610754219Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.610865922Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.611226409Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.611248896Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.611262783Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.611443957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.611461844Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:43.611474862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.611044466Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_connection connected from Ok(127.0.0.1:57011)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.61114924Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"io task","client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.612077123Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.612105258Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.612773489Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613409475Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613437341Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[0]R flush_numbers[0..12]: [388, 386, 354, 378, 378, 387, 387, 378, 378, 386, 385, 387]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613453729Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[0]R generation[0..12]: [189, 187, 173, 184, 184, 188, 188, 184, 184, 187, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613468686Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613482944Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[1]R flush_numbers[0..12]: [388, 386, 354, 378, 378, 387, 387, 378, 378, 386, 385, 387]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613496652Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[1]R generation[0..12]: [189, 187, 173, 184, 184, 188, 188, 184, 184, 187, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.61351067Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613524708Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[2]R flush_numbers[0..12]: [388, 386, 354, 378, 378, 387, 387, 378, 378, 386, 385, 387]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613538386Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[2]R generation[0..12]: [189, 187, 173, 184, 184, 188, 188, 184, 184, 187, 186, 188]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613552074Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613566062Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":Wait for a query_work_queue command to finish before sending IO 1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613604566Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613626613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Next flush: 389","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.61364484Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613657928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"":"downstairs","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613670946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613682785Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"0","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613695913Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"1","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613708831Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"client":"2","":"downstairs","session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613721969Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613733957Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"6ca606cd-1215-47b4-ae92-e623b8869895 is now active with session: 044c1c73-2001-4aba-b3c2-eccf9f6dd101","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613746595Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613758613Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943,"session_id":"044c1c73-2001-4aba-b3c2-eccf9f6dd101"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:46:53.613770762Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 426 len 20480 data: 13 16 14 16 17 02/30 Read block 243 len 40960 03/30 Write block 573 len 24576 data: 23 22 24 21 16 17 04/30 Write block 476 len 12288 data: 17 17 19 05/30 Write block 55 len 36864 data: 15 14 15 14 13 17 17 17 17 06/30 Write block 112 len 20480 data: 24 26 21 19 19 07/30 Write block 562 len 40960 data: 17 17 14 13 17 17 19 17 18 23 08/30 Flush 09/30 Write block 191 len 28672 data: 12 14 11 11 14 9 9 10/30 Write block 363 len 16384 data: 12 12 15 18 11/30 Read block 473 len 12288 12/30 Read block 145 len 32768 13/30 Read block 256 len 36864 14/30 Read block 136 len 8192 15/30 Write block 454 len 32768 data: 10 10 9 9 9 8 8 9 16/30 Write block 373 len 4096 data: 14 17/30 Read block 322 len 32768 18/30 Write block 51 len 20480 data: 13 13 14 12 16 19/30 Read block 378 len 28672 20/30 Write block 384 len 16384 data: 14 13 13 11 21/30 Flush 22/30 Write block 230 len 32768 data: 11 9 11 11 9 7 9 9 23/30 Read block 541 len 24576 24/30 Read block 579 len 32768 25/30 Read block 523 len 8192 26/30 Read block 142 len 28672 27/30 Write block 542 len 40960 data: 8 9 8 9 9 10 10 9 10 8 28/30 Read block 352 len 8192 29/30 Read block 422 len 36864 30/30 Write block 183 len 4096 data: 11 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 23 Acked 1022 Write 8 Done Done Done false 24 Acked 1023 Read 6 Done Done Done false 25 Acked 1024 Read 8 Done Done Done false 26 Acked 1025 Read 2 Done Done Done false 27 Acked 1026 Read 7 Done Done Done false 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Read 2 Done Done Done false 30 Acked 1029 Read 9 Done Done Done false 31 Acked 1030 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:46:53.641175449Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1943} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 388 388 388 F F F 1 020-039 187 187 187 386 386 386 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 190 190 190 389 389 389 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 190 190 190 390 390 390 T T T 10 200-219 186 186 186 385 385 385 F F F 11 220-239 188 188 188 387 387 387 T T T 12 240-259 184 184 184 379 379 379 F F F 13 260-279 187 187 187 386 386 386 F F F 14 280-299 189 189 189 388 388 388 F F F 15 300-319 189 189 189 388 388 388 F F F 16 320-339 188 188 188 387 387 387 F F F 17 340-359 187 187 187 386 386 386 F F F 18 360-379 190 190 190 390 390 390 F F F 19 380-399 190 190 190 391 391 391 F F F 20 400-419 188 188 188 387 387 387 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 190 190 190 390 390 390 F F F 23 460-479 190 190 190 390 390 390 F F F 24 480-499 188 188 188 387 387 387 F F F 25 500-519 188 188 188 387 387 387 F F F 26 520-539 188 188 188 387 387 387 F F F 27 540-559 188 188 188 387 387 387 T T T 28 560-579 190 190 190 389 389 389 F F F 29 580-599 188 188 188 387 387 387 F F F Max gen: 190, Max flush: 391 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 189 189 189 388 388 388 F F F 1 020-039 187 187 187 386 386 386 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 190 190 190 389 389 389 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 191 191 191 392 392 392 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 191 191 191 392 392 392 F F F 12 240-259 184 184 184 379 379 379 F F F 13 260-279 187 187 187 386 386 386 F F F 14 280-299 189 189 189 388 388 388 F F F 15 300-319 189 189 189 388 388 388 F F F 16 320-339 188 188 188 387 387 387 F F F 17 340-359 187 187 187 386 386 386 F F F 18 360-379 190 190 190 390 390 390 F F F 19 380-399 190 190 190 391 391 391 F F F 20 400-419 188 188 188 387 387 387 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 190 190 190 390 390 390 F F F 23 460-479 190 190 190 390 390 390 F F F 24 480-499 188 188 188 387 387 387 F F F 25 500-519 188 188 188 387 387 387 F F F 26 520-539 188 188 188 387 387 387 F F F 27 540-559 191 191 191 392 392 392 F F F 28 560-579 190 190 190 389 389 389 F F F 29 580-599 188 188 188 387 387 387 F F F Max gen: 191, Max flush: 392 Begin loop 96 Downstairs to restart: 1 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.840302549Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.840801226Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.840831901Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951} {"msg":"Crucible 22f9a284-4ebc-4f74-8551-330884941f12 has session id: b0a6cfd5-bc36-42f3-90f9-87323072b1ef","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841069926Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Upstairs opts: Upstairs UUID: 22f9a284-4ebc-4f74-8551-330884941f12, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.8411078Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Crucible stats registered with UUID: 22f9a284-4ebc-4f74-8551-330884941f12","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841123848Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841411595Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951} {"msg":"22f9a284-4ebc-4f74-8551-330884941f12 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841567272Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841589609Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841614115Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841640181Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841664318Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841686624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841709471Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841732257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841754684Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841775321Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:53673)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.841989889Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_connection connect to 127.0.0.1:8820 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:07.84212003Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:07.842159504Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"downstairs task for 1 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:07.84219054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842214106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842242442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:64870)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842268928Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842359215Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842843253Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842881287Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842909903Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.842936029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.843480098Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.843588252Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.844117754Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.84414421Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.844168476Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.844186444Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.844200152Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:07.8442131Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.842908283Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_connection connected from Ok(127.0.0.1:61953)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.842976203Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"io task","client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.844149429Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.844188464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.844723424Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845331724Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.84535943Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[0]R flush_numbers[0..12]: [388, 386, 390, 389, 378, 389, 387, 378, 378, 392, 385, 392]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845376448Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[0]R generation[0..12]: [189, 187, 190, 190, 184, 190, 188, 184, 184, 191, 186, 191]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845391535Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845405723Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[1]R flush_numbers[0..12]: [388, 386, 390, 389, 378, 389, 387, 378, 378, 392, 385, 392]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845419601Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[1]R generation[0..12]: [189, 187, 190, 190, 184, 190, 188, 184, 184, 191, 186, 191]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845433299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845447377Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[2]R flush_numbers[0..12]: [388, 386, 390, 389, 378, 389, 387, 378, 378, 392, 385, 392]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845460975Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"[2]R generation[0..12]: [189, 187, 190, 190, 184, 190, 188, 184, 184, 191, 186, 191]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845474463Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.8454956Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845509638Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845523066Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Next flush: 393","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845535774Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845548642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"":"downstairs","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.84556216Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845574088Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"0","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845587426Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"1","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845600454Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"client":"2","":"downstairs","session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845613612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.84562568Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"22f9a284-4ebc-4f74-8551-330884941f12 is now active with session: b0a6cfd5-bc36-42f3-90f9-87323072b1ef","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845638299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845650107Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951,"session_id":"b0a6cfd5-bc36-42f3-90f9-87323072b1ef"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:17.845662145Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 190 len 12288 data: 12 13 15 02/30 Read block 100 len 32768 03/30 Read block 434 len 40960 04/30 Read block 329 len 12288 05/30 Write block 298 len 4096 data: 17 06/30 Write block 16 len 16384 data: 12 12 9 10 07/30 Read block 100 len 24576 08/30 Write block 335 len 4096 data: 13 09/30 Read block 200 len 36864 10/30 Read block 56 len 32768 11/30 Write block 525 len 28672 data: 13 12 12 10 9 7 10 12/30 Write block 259 len 4096 data: 19 13/30 Write block 255 len 32768 data: 15 15 17 21 20 16 16 13 14/30 Read block 311 len 32768 15/30 Read block 109 len 4096 16/30 Read block 92 len 28672 17/30 Write block 181 len 12288 data: 13 11 12 18/30 Read block 471 len 24576 19/30 Write block 185 len 28672 data: 9 10 12 12 13 13 14 20/30 Write block 454 len 32768 data: 11 11 10 10 10 9 9 10 21/30 Read block 48 len 40960 22/30 Write block 70 len 32768 data: 13 11 11 9 8 7 10 12 23/30 Write block 486 len 36864 data: 25 21 24 24 22 18 17 16 19 24/30 Read block 176 len 12288 25/30 Read block 6 len 20480 26/30 Write block 545 len 36864 data: 10 10 11 11 10 11 9 6 8 27/30 Write block 326 len 8192 data: 19 17 28/30 Write block 35 len 12288 data: 8 8 7 29/30 Read block 440 len 16384 30/30 Write block 507 len 36864 data: 15 17 21 20 18 17 16 20 22 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 27 Acked 1026 Write 9 Done Done Done false 28 Acked 1027 Write 2 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Read 4 Done Done Done false 31 Acked 1030 Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 31 30 26 29 28 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:17.869667847Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1951} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 F F F 1 020-039 187 187 187 386 386 386 T T T 2 040-059 190 190 190 390 390 390 F F F 3 060-079 192 192 192 393 393 393 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 192 192 192 393 393 393 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 191 191 191 392 392 392 F F F 12 240-259 192 192 192 393 393 393 F F F 13 260-279 192 192 192 393 393 393 F F F 14 280-299 192 192 192 393 393 393 F F F 15 300-319 189 189 189 388 388 388 F F F 16 320-339 192 192 192 393 393 393 T T T 17 340-359 187 187 187 386 386 386 F F F 18 360-379 190 190 190 390 390 390 F F F 19 380-399 190 190 190 391 391 391 F F F 20 400-419 188 188 188 387 387 387 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 192 192 192 393 393 393 F F F 23 460-479 192 192 192 393 393 393 F F F 24 480-499 192 192 192 393 393 393 F F F 25 500-519 188 188 188 387 387 387 T T T 26 520-539 192 192 192 393 393 393 F F F 27 540-559 191 191 191 392 392 392 T T T 28 560-579 190 190 190 389 389 389 F F F 29 580-599 188 188 188 387 387 387 F F F Max gen: 192, Max flush: 393 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 F F F 1 020-039 193 193 193 394 394 394 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 192 192 192 393 393 393 F F F 4 080-099 184 184 184 378 378 378 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 184 184 184 378 378 378 F F F 8 160-179 184 184 184 378 378 378 F F F 9 180-199 192 192 192 393 393 393 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 191 191 191 392 392 392 F F F 12 240-259 192 192 192 393 393 393 F F F 13 260-279 192 192 192 393 393 393 F F F 14 280-299 192 192 192 393 393 393 F F F 15 300-319 189 189 189 388 388 388 F F F 16 320-339 193 193 193 394 394 394 F F F 17 340-359 187 187 187 386 386 386 F F F 18 360-379 190 190 190 390 390 390 F F F 19 380-399 190 190 190 391 391 391 F F F 20 400-419 188 188 188 387 387 387 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 192 192 192 393 393 393 F F F 23 460-479 192 192 192 393 393 393 F F F 24 480-499 192 192 192 393 393 393 F F F 25 500-519 193 193 193 394 394 394 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 193 193 193 394 394 394 F F F 28 560-579 190 190 190 389 389 389 F F F 29 580-599 188 188 188 387 387 387 F F F Max gen: 193, Max flush: 394 Begin loop 97 Downstairs to restart: 0 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.082476088Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083091077Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083123042Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959} {"msg":"Crucible 3ddf2eab-2e2e-4cca-89b8-2b4000887eb0 has session id: 87848183-e058-4594-998e-9d067ba93d48","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083165396Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Upstairs opts: Upstairs UUID: 3ddf2eab-2e2e-4cca-89b8-2b4000887eb0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083200131Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Crucible stats registered with UUID: 3ddf2eab-2e2e-4cca-89b8-2b4000887eb0","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083222788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083528672Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959} {"msg":"3ddf2eab-2e2e-4cca-89b8-2b4000887eb0 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083634667Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083659013Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083674571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083688339Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083709445Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083733282Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.08374839Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083761998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083775946Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.083789624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_connection connect to 127.0.0.1:8810 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:32.083923434Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:32.083981465Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_connection connected from Ok(127.0.0.1:39561)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084011761Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"downstairs task for 0 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:32.084096368Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_connection connected from Ok(127.0.0.1:40725)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084130573Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084158959Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084173927Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084258084Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084811872Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084847087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084871093Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.084940933Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.085361061Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.085470654Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.085821962Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.085850998Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.085876604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.085994867Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.086013834Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:32.086027262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.085191185Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_connection connected from Ok(127.0.0.1:57589)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.085249157Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"io task","client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.086370571Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.086399737Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087023254Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087618016Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087646142Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[0]R flush_numbers[0..12]: [393, 394, 390, 393, 378, 389, 387, 378, 378, 393, 385, 392]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087663029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[0]R generation[0..12]: [192, 193, 190, 192, 184, 190, 188, 184, 184, 192, 186, 191]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087677657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087692055Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[1]R flush_numbers[0..12]: [393, 394, 390, 393, 378, 389, 387, 378, 378, 393, 385, 392]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087705883Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[1]R generation[0..12]: [192, 193, 190, 192, 184, 190, 188, 184, 184, 192, 186, 191]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087719701Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087733649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[2]R flush_numbers[0..12]: [393, 394, 390, 393, 378, 389, 387, 378, 378, 393, 385, 392]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087747387Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[2]R generation[0..12]: [192, 193, 190, 192, 184, 190, 188, 184, 184, 192, 186, 191]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087766184Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087781162Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087798329Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087811487Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Next flush: 395","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087824265Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087837033Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"":"downstairs","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087850202Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.08786209Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"0","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087875618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"1","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087888576Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"client":"2","":"downstairs","session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087901634Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087913862Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"3ddf2eab-2e2e-4cca-89b8-2b4000887eb0 is now active with session: 87848183-e058-4594-998e-9d067ba93d48","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.08792658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087938618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959,"session_id":"87848183-e058-4594-998e-9d067ba93d48"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:42.087950867Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 4096 data: 10 02/30 Read block 259 len 12288 03/30 Write block 96 len 4096 data: 14 04/30 Write block 142 len 36864 data: 14 15 15 18 13 9 11 11 12 05/30 Write block 34 len 16384 data: 8 9 9 8 06/30 Write block 339 len 20480 data: 15 15 14 14 14 07/30 Write block 476 len 8192 data: 18 18 08/30 Read block 375 len 4096 09/30 Read block 333 len 8192 10/30 Write block 173 len 28672 data: 15 15 17 12 13 11 10 11/30 Read block 190 len 40960 12/30 Read block 350 len 4096 13/30 Write block 223 len 12288 data: 11 8 9 14/30 Write block 373 len 12288 data: 15 15 15 15/30 Flush 16/30 Write block 491 len 28672 data: 19 18 17 20 17 19 22 17/30 Write block 401 len 28672 data: 19 20 18 19 22 22 19 18/30 Flush 19/30 Flush 20/30 Write block 450 len 20480 data: 9 8 9 11 12 21/30 Read block 22 len 32768 22/30 Read block 507 len 40960 23/30 Read block 191 len 20480 24/30 Write block 70 len 20480 data: 14 12 12 10 9 25/30 Read block 535 len 28672 26/30 Write block 574 len 40960 data: 23 25 22 17 18 15 18 19 17 12 27/30 Write block 296 len 28672 data: 16 18 18 20 18 16 15 28/30 Write block 273 len 16384 data: 15 16 18 19 29/30 Read block 336 len 24576 30/30 Write block 586 len 4096 data: 13 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 5 Done Done Done false 21 Acked 1020 Read 8 Done Done Done false 22 Acked 1021 Read 10 Done Done Done false 23 Acked 1022 Read 5 Done Done Done false 24 Acked 1023 Write 5 Done Done Done false 25 Acked 1024 Read 7 Done Done Done false 26 Acked 1025 Write 10 Done Done Done false 27 Acked 1026 Write 7 Done Done Done false 28 Acked 1027 Write 4 Done Done Done false 29 Acked 1028 Read 6 Done Done Done false 30 Acked 1029 Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:42.114422643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1959} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 F F F 1 020-039 194 194 194 395 395 395 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 192 192 192 393 393 393 T T T 4 080-099 194 194 194 395 395 395 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 194 194 194 395 395 395 F F F 8 160-179 194 194 194 395 395 395 F F F 9 180-199 192 192 192 393 393 393 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 194 194 194 395 395 395 F F F 12 240-259 192 192 192 393 393 393 F F F 13 260-279 192 192 192 393 393 393 T T T 14 280-299 192 192 192 393 393 393 T T T 15 300-319 189 189 189 388 388 388 T T T 16 320-339 194 194 194 395 395 395 F F F 17 340-359 194 194 194 395 395 395 F F F 18 360-379 194 194 194 395 395 395 F F F 19 380-399 190 190 190 391 391 391 F F F 20 400-419 194 194 194 396 396 396 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 192 192 192 393 393 393 T T T 23 460-479 194 194 194 395 395 395 F F F 24 480-499 194 194 194 396 396 396 F F F 25 500-519 193 193 193 394 394 394 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 193 193 193 394 394 394 F F F 28 560-579 190 190 190 389 389 389 T T T 29 580-599 188 188 188 387 387 387 T T T Max gen: 194, Max flush: 396 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 F F F 1 020-039 194 194 194 395 395 395 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 195 195 195 397 397 397 F F F 4 080-099 194 194 194 395 395 395 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 188 188 188 387 387 387 F F F 7 140-159 194 194 194 395 395 395 F F F 8 160-179 194 194 194 395 395 395 F F F 9 180-199 192 192 192 393 393 393 F F F 10 200-219 186 186 186 385 385 385 F F F 11 220-239 194 194 194 395 395 395 F F F 12 240-259 192 192 192 393 393 393 F F F 13 260-279 195 195 195 397 397 397 F F F 14 280-299 195 195 195 397 397 397 F F F 15 300-319 195 195 195 397 397 397 F F F 16 320-339 194 194 194 395 395 395 F F F 17 340-359 194 194 194 395 395 395 F F F 18 360-379 194 194 194 395 395 395 F F F 19 380-399 190 190 190 391 391 391 F F F 20 400-419 194 194 194 396 396 396 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 195 195 195 397 397 397 F F F 23 460-479 194 194 194 395 395 395 F F F 24 480-499 194 194 194 396 396 396 F F F 25 500-519 193 193 193 394 394 394 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 193 193 193 394 394 394 F F F 28 560-579 195 195 195 397 397 397 F F F 29 580-599 195 195 195 397 397 397 F F F Max gen: 195, Max flush: 397 Begin loop 98 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.361367005Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.361884119Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.361912564Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967} {"msg":"Crucible 6f4f7730-01a7-4e43-9466-63a7c72f6eb9 has session id: edcab0de-15ac-454c-869a-1fa3d25cc9c1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.361934461Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Upstairs opts: Upstairs UUID: 6f4f7730-01a7-4e43-9466-63a7c72f6eb9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.361967076Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Crucible stats registered with UUID: 6f4f7730-01a7-4e43-9466-63a7c72f6eb9","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362013649Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362353539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967} {"msg":"6f4f7730-01a7-4e43-9466-63a7c72f6eb9 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362426248Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362452624Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362470442Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.36248453Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362498268Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362511436Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362524694Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362537692Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.3625521Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362565827Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46638)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362804642Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:55949)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.362843257Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:56.363031658Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:56.363071643Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:47:56.363095049Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363108317Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363121615Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363188205Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363675683Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363701539Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363716117Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.363729495Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364355622Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364384618Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364930987Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364957593Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364972161Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364984319Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.364997657Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:47:56.365014464Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.364681604Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_connection connected from Ok(127.0.0.1:59842)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.364750334Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"io task","client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.365737907Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.365769243Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366321681Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366876619Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366900385Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[0]R flush_numbers[0..12]: [393, 395, 390, 397, 395, 389, 387, 395, 395, 393, 385, 395]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366920982Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[0]R generation[0..12]: [192, 194, 190, 195, 194, 190, 188, 194, 194, 192, 186, 194]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.36693589Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366950398Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[1]R flush_numbers[0..12]: [393, 395, 390, 397, 395, 389, 387, 395, 395, 393, 385, 395]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366964146Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[1]R generation[0..12]: [192, 194, 190, 195, 194, 190, 188, 194, 194, 192, 186, 194]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366978033Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.366992262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[2]R flush_numbers[0..12]: [393, 395, 390, 397, 395, 389, 387, 395, 395, 393, 385, 395]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367011299Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[2]R generation[0..12]: [192, 194, 190, 195, 194, 190, 188, 194, 194, 192, 186, 194]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367025886Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:48:06.367040054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367055092Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.36706845Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Next flush: 398","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367081208Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367094106Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"":"downstairs","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367107934Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367128941Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"0","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367154597Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"1","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367173245Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"client":"2","":"downstairs","session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367187262Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.36720522Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"6f4f7730-01a7-4e43-9466-63a7c72f6eb9 is now active with session: edcab0de-15ac-454c-869a-1fa3d25cc9c1","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367228806Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367250773Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967,"session_id":"edcab0de-15ac-454c-869a-1fa3d25cc9c1"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:06.367275689Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 132 len 8192 data: 13 14 03/30 Flush 04/30 Write block 490 len 36864 data: 23 20 19 18 21 18 20 23 20 05/30 Write block 362 len 36864 data: 11 13 13 16 19 22 20 18 17 06/30 Read block 230 len 4096 07/30 Read block 9 len 4096 08/30 Flush 09/30 Write block 510 len 28672 data: 21 19 18 17 21 23 21 10/30 Read block 139 len 12288 11/30 Read block 216 len 36864 12/30 Read block 202 len 40960 13/30 Write block 129 len 16384 data: 11 14 14 14 14/30 Write block 578 len 12288 data: 19 16 19 15/30 Write block 375 len 28672 data: 16 12 14 14 17 19 17 16/30 Write block 230 len 20480 data: 12 10 12 12 10 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 196 len 32768 data: 10 10 8 13 14 12 15 15 21/30 Write block 254 len 16384 data: 21 16 16 18 22/30 Read block 340 len 12288 23/30 Read block 55 len 28672 24/30 Read block 486 len 12288 25/30 Read block 273 len 12288 26/30 Write block 546 len 28672 data: 11 12 12 11 12 10 7 27/30 Read block 508 len 40960 28/30 Read block 74 len 16384 29/30 Write block 225 len 36864 data: 10 8 8 9 12 13 11 13 13 30/30 Write block 564 len 20480 data: 15 14 18 18 20 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 20 Acked 1019 Write 8 Done Done Done false 21 Acked 1020 Write 4 Done Done Done false 22 Acked 1021 Read 3 Done Done Done false 23 Acked 1022 Read 7 Done Done Done false 24 Acked 1023 Read 3 Done Done Done false 25 Acked 1024 Read 3 Done Done Done false 26 Acked 1025 Write 7 Done Done Done false 27 Acked 1026 Read 10 Done Done Done false 28 Acked 1027 Read 4 Done Done Done false 29 Acked 1028 Write 9 Done Done Done false 30 Acked 1029 Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 30 29 28 27 26 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:06.394433054Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1967} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 F F F 1 020-039 194 194 194 395 395 395 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 195 195 195 397 397 397 F F F 4 080-099 194 194 194 395 395 395 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 196 196 196 401 401 401 F F F 7 140-159 194 194 194 395 395 395 F F F 8 160-179 194 194 194 395 395 395 F F F 9 180-199 192 192 192 393 393 393 T T T 10 200-219 186 186 186 385 385 385 T T T 11 220-239 196 196 196 401 401 401 T T T 12 240-259 192 192 192 393 393 393 T T T 13 260-279 195 195 195 397 397 397 F F F 14 280-299 195 195 195 397 397 397 F F F 15 300-319 195 195 195 397 397 397 F F F 16 320-339 194 194 194 395 395 395 F F F 17 340-359 194 194 194 395 395 395 F F F 18 360-379 196 196 196 401 401 401 F F F 19 380-399 196 196 196 401 401 401 F F F 20 400-419 194 194 194 396 396 396 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 195 195 195 397 397 397 F F F 23 460-479 194 194 194 395 395 395 F F F 24 480-499 196 196 196 400 400 400 F F F 25 500-519 196 196 196 401 401 401 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 193 193 193 394 394 394 T T T 28 560-579 196 196 196 401 401 401 T T T 29 580-599 196 196 196 401 401 401 F F F Max gen: 196, Max flush: 401 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 F F F 1 020-039 194 194 194 395 395 395 F F F 2 040-059 190 190 190 390 390 390 F F F 3 060-079 195 195 195 397 397 397 F F F 4 080-099 194 194 194 395 395 395 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 196 196 196 401 401 401 F F F 7 140-159 194 194 194 395 395 395 F F F 8 160-179 194 194 194 395 395 395 F F F 9 180-199 197 197 197 402 402 402 F F F 10 200-219 197 197 197 402 402 402 F F F 11 220-239 197 197 197 402 402 402 F F F 12 240-259 197 197 197 402 402 402 F F F 13 260-279 195 195 195 397 397 397 F F F 14 280-299 195 195 195 397 397 397 F F F 15 300-319 195 195 195 397 397 397 F F F 16 320-339 194 194 194 395 395 395 F F F 17 340-359 194 194 194 395 395 395 F F F 18 360-379 196 196 196 401 401 401 F F F 19 380-399 196 196 196 401 401 401 F F F 20 400-419 194 194 194 396 396 396 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 195 195 195 397 397 397 F F F 23 460-479 194 194 194 395 395 395 F F F 24 480-499 196 196 196 400 400 400 F F F 25 500-519 196 196 196 401 401 401 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 197 197 197 402 402 402 F F F 28 560-579 197 197 197 402 402 402 F F F 29 580-599 196 196 196 401 401 401 F F F Max gen: 197, Max flush: 402 Begin loop 99 Downstairs to restart: 2 {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.629719696Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"955d5f2fb08fa12300fcdaa7dafdc9edb7db9942\",\n git_commit_timestamp: \"2024-08-01T19:59:22.000000000Z\",\n git_branch: \"fioffs-redux\",\n rustc_semver: \"1.76.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"07dca489ac2d933c78d3c5158e3f43beefeb02ce\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630551333Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975} {"msg":"Upstairs <-> Downstairs Message Version: 11","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630584528Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975} {"msg":"Crucible f8e01280-9956-4f3e-ac9e-295590f64936 has session id: 9f6bc81a-56fd-404e-a988-09c31670d028","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630609674Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Upstairs opts: Upstairs UUID: f8e01280-9956-4f3e-ac9e-295590f64936, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630646488Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Crucible stats registered with UUID: f8e01280-9956-4f3e-ac9e-295590f64936","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630678164Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} Crucible runtime is spawned {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630858417Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975} {"msg":"f8e01280-9956-4f3e-ac9e-295590f64936 active request set","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630926247Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.630948983Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.63097527Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631001936Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631025952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631050019Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631073575Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631098811Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631125297Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631149864Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_connection connect to 127.0.0.1:8830 failure: Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:20.631285953Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"client task is sending Done(ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" }))","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:20.631337036Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_connection connected from Ok(127.0.0.1:41235)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631366612Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"downstairs task for 2 stopped due to ConnectionFailed(Os { code: 146, kind: ConnectionRefused, message: \"Connection refused\" })","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:20.631391978Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Gone missing, transition from New to Disconnected","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631413045Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Disconnected -> New","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631435551Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"client task is exiting","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.631457818Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_connection connected from Ok(127.0.0.1:48399)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.63158065Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.632166483Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.632202788Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.632231273Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.632258029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.632847952Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.632885027Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.633347298Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.633373294Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.633396171Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.63346739Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.633487037Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:20.633500895Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.632035632Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_connection connected from Ok(127.0.0.1:50048)","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.632112451Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"io task","client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.633247063Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.633276349Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.633799711Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634383004Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634416879Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[0]R flush_numbers[0..12]: [393, 395, 390, 397, 395, 389, 401, 395, 395, 402, 402, 402]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634433367Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[0]R generation[0..12]: [192, 194, 190, 195, 194, 190, 196, 194, 194, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634448035Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634462193Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[1]R flush_numbers[0..12]: [393, 395, 390, 397, 395, 389, 401, 395, 395, 402, 402, 402]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634476151Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[1]R generation[0..12]: [192, 194, 190, 195, 194, 190, 196, 194, 194, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634490029Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634508046Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[2]R flush_numbers[0..12]: [393, 395, 390, 397, 395, 389, 401, 395, 395, 402, 402, 402]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634522544Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[2]R generation[0..12]: [192, 194, 190, 195, 194, 190, 196, 194, 194, 197, 197, 197]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634536602Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.6345506Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-08-05T19:48:30.634563518Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634579915Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Next flush: 403","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634592973Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634605931Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"":"downstairs","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634618979Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634630868Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"0","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634644026Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"1","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634656964Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"client":"2","":"downstairs","session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634670002Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.63468213Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"f8e01280-9956-4f3e-ac9e-295590f64936 is now active with session: 9f6bc81a-56fd-404e-a988-09c31670d028","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634694968Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634709536Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975,"session_id":"9f6bc81a-56fd-404e-a988-09c31670d028"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-08-05T19:48:30.634721914Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975} Region: es:20 ec:30 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 123 len 40960 data: 13 11 11 11 11 9 12 15 15 15 02/30 Read block 557 len 28672 03/30 Write block 414 len 16384 data: 11 10 12 10 04/30 Read block 130 len 8192 05/30 Read block 245 len 20480 06/30 Write block 336 len 36864 data: 14 15 14 16 16 15 15 15 14 07/30 Read block 287 len 16384 08/30 Read block 378 len 28672 09/30 Read block 589 len 36864 10/30 Write block 276 len 36864 data: 20 18 15 15 16 14 13 16 17 11/30 Write block 394 len 32768 data: 18 19 21 21 18 16 19 20 12/30 Read block 31 len 12288 13/30 Write block 387 len 4096 data: 12 14/30 Write block 78 len 40960 data: 14 14 14 16 16 17 16 12 9 8 15/30 Read block 281 len 12288 16/30 Write block 164 len 32768 data: 21 24 22 23 20 22 21 19 17/30 Write block 191 len 36864 data: 15 16 12 12 15 11 11 9 14 18/30 Read block 189 len 36864 19/30 Write block 570 len 8192 data: 19 24 20/30 Read block 394 len 28672 21/30 Write block 397 len 36864 data: 22 19 17 20 21 21 19 20 23 22/30 Read block 25 len 4096 23/30 Write block 38 len 20480 data: 6 7 9 11 14 24/30 Read block 188 len 40960 25/30 Read block 34 len 4096 26/30 Flush 27/30 Write block 13 len 40960 data: 13 11 13 13 13 10 11 11 10 11 28/30 Write block 206 len 12288 data: 21 23 20 29/30 Write block 127 len 4096 data: 12 30/30 Read block 307 len 12288 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active GW_ID ACK DSID TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 28 Acked 1027 Write 10 Done Done Done false 29 Acked 1028 Write 3 Done Done Done false 30 Acked 1029 Write 1 Done Done Sent false 31 Acked 1030 Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL New 0 0 0 0 Sent 1 0 2 3 Done 3 4 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1026 1026 1026 Downstairs last five completed: 1026 1025 1024 1023 1022 Upstairs last five completed: 31 30 29 28 27 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-08-05T19:48:30.662702087Z","hostname":"w-01J4HZ047PSWY754NCCNYYNCWV","pid":1975} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 192 192 192 393 393 393 T T T 1 020-039 198 198 198 404 404 404 T T T 2 040-059 198 198 198 404 404 404 F F F 3 060-079 198 198 198 403 403 403 F F F 4 080-099 198 198 198 403 403 403 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 198 198 198 403 403 403 T T T 7 140-159 194 194 194 395 395 395 F F F 8 160-179 198 198 198 403 403 403 F F F 9 180-199 198 198 198 403 403 403 F F F 10 200-219 197 197 197 402 402 402 T T T 11 220-239 197 197 197 402 402 402 F F F 12 240-259 197 197 197 402 402 402 F F F 13 260-279 198 198 198 403 403 403 F F F 14 280-299 198 198 198 403 403 403 F F F 15 300-319 195 195 195 397 397 397 F F F 16 320-339 198 198 198 403 403 403 F F F 17 340-359 198 198 198 403 403 403 F F F 18 360-379 196 196 196 401 401 401 F F F 19 380-399 198 198 198 403 403 403 F F F 20 400-419 198 198 198 403 403 403 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 195 195 195 397 397 397 F F F 23 460-479 194 194 194 395 395 395 F F F 24 480-499 196 196 196 400 400 400 F F F 25 500-519 196 196 196 401 401 401 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 197 197 197 402 402 402 F F F 28 560-579 198 198 198 403 403 403 F F F 29 580-599 196 196 196 401 401 401 F F F Max gen: 198, Max flush: 404 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 199 199 199 405 405 405 F F F 1 020-039 199 199 199 405 405 405 F F F 2 040-059 198 198 198 404 404 404 F F F 3 060-079 198 198 198 403 403 403 F F F 4 080-099 198 198 198 403 403 403 F F F 5 100-119 190 190 190 389 389 389 F F F 6 120-139 199 199 199 405 405 405 F F F 7 140-159 194 194 194 395 395 395 F F F 8 160-179 198 198 198 403 403 403 F F F 9 180-199 198 198 198 403 403 403 F F F 10 200-219 199 199 199 405 405 405 F F F 11 220-239 197 197 197 402 402 402 F F F 12 240-259 197 197 197 402 402 402 F F F 13 260-279 198 198 198 403 403 403 F F F 14 280-299 198 198 198 403 403 403 F F F 15 300-319 195 195 195 397 397 397 F F F 16 320-339 198 198 198 403 403 403 F F F 17 340-359 198 198 198 403 403 403 F F F 18 360-379 196 196 196 401 401 401 F F F 19 380-399 198 198 198 403 403 403 F F F 20 400-419 198 198 198 403 403 403 F F F 21 420-439 190 190 190 389 389 389 F F F 22 440-459 195 195 195 397 397 397 F F F 23 460-479 194 194 194 395 395 395 F F F 24 480-499 196 196 196 400 400 400 F F F 25 500-519 196 196 196 401 401 401 F F F 26 520-539 192 192 192 393 393 393 F F F 27 540-559 197 197 197 402 402 402 F F F 28 560-579 198 198 198 403 403 403 F F F 29 580-599 196 196 196 401 401 401 F F F Max gen: 199, Max flush: 405 40:03 Test duration Test completed real 40:03.757845988 user 39.372762673 sys 18.511955629 trap 0.053071916 tflt 0.145874608 dflt 0.139512467 kflt 0.008459334 lock 63:31:10.645189104 slp 3:17:21.605517277 lat 8.256287389 stop 0.062703355