Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.330366 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.021012627 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.018970456 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.86998664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.951674421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.953185334Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.9532892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.953308157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.953322745Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} {"msg":"Crucible 549c9add-7223-4709-81e1-ece5cfcaa8f7 has session id: beac44a7-8b47-449d-8fa3-a3184e1a3337","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.954295362Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Upstairs opts: Upstairs UUID: 549c9add-7223-4709-81e1-ece5cfcaa8f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.954479147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Crucible stats registered with UUID: 549c9add-7223-4709-81e1-ece5cfcaa8f7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.954507693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987041095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987099827Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} {"msg":"549c9add-7223-4709-81e1-ece5cfcaa8f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987168367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987200133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987247486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987263114Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987276673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987289551Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987302679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987315577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987329025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987342124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_connection connected from Ok(127.0.0.1:41128)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987913355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_connection connected from Ok(127.0.0.1:52503)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.98795159Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_connection connected from Ok(127.0.0.1:44178)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.987978486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.989732567Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.989770632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.989828134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.989867398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.989899734Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.98992662Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.990811659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.990847584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.99087646Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991763389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991795875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991811912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991824611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991838269Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991851487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991863265Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991876184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991944854Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991964721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991981939Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.991995407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992008466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992022214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992035482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.9920485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992062248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992074857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992087725Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992100273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992229735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"":"downstairs","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992248313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992260361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"0","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992275009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"1","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992288297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"client":"2","":"downstairs","session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992303845Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992316004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"549c9add-7223-4709-81e1-ece5cfcaa8f7 is now active with session: beac44a7-8b47-449d-8fa3-a3184e1a3337","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992328602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.9923402Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188,"session_id":"beac44a7-8b47-449d-8fa3-a3184e1a3337"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:17.992352048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:22.092146066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1188} Begin loop 1 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.126251472Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.172759123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.173392417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.173433341Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.173452538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.173467136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} {"msg":"Crucible 031b4bcf-9285-468b-bee4-ba6734d3fc84 has session id: d575ed91-1780-4e88-9607-c6371c03ee0e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.174066235Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Upstairs opts: Upstairs UUID: 031b4bcf-9285-468b-bee4-ba6734d3fc84, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.1741008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Crucible stats registered with UUID: 031b4bcf-9285-468b-bee4-ba6734d3fc84","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.174116888Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206002808Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206035923Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} {"msg":"031b4bcf-9285-468b-bee4-ba6734d3fc84 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206111223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.2061335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206163056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206180743Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206209839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206241155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206266332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206292838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206332563Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206366838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48936)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206398034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60979)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206431469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41741)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.206592177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207237579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207263635Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207277973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207291612Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.20730508Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207328447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207902538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.207935474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.2079618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208529033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208570027Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208596633Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208612201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208627049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208640137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208652016Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208664944Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208678192Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.20869164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208705558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208718696Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208731995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Wait for a query_work_queue command to finish before sending IO [1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208752632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208776859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208799396Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208823082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208847809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208864836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208877665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208890443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"":"downstairs","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208904171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.20891588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"0","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208928878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"1","":"downstairs","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208941816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"client":"2","":"downstairsDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208958814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208971082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"031b4bcf-9285-468b-bee4-ba6734d3fc84 is now active with session: d575ed91-1780-4e88-9607-c6371c03ee0e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.20898369Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.208995469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190,"session_id":"d575ed91-1780-4e88-9607-c6371c03ee0e"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:22.209007717Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 554 len 32768 data: 2 2 2 2 2 2 2 2 02/30 Read block 47 len 24576 03/30 Read block 254 len 4096 04/30 Flush 05/30 Write block 26 len 20480 data: 2 2 2 2 2 06/30 Write block 16 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 143 len 32768 08/30 Flush 09/30 Flush 10/30 Write block 544 len 16384 data: 2 2 2 2 11/30 Read block 534 len 40960 12/30 Read block 449 len 4096 13/30 Read block 176 len 16384 14/30 Read block 513 len 28672 15/30 Read block 540 len 28672 16/30 Write block 135 len 12288 data: 2 2 2 17/30 Read block 550 len 32768 18/30 Write block 467 len 16384 data: 2 2 2 2 19/30 Read block 267 len 12288 20/30 Write block 499 len 12288 data: 2 2 2 21/30 Write block 288 len 24576 data: 2 2 2 2 2 2 22/30 Flush 23/30 Write block 90 len 24576 data: 2 2 2 2 2 2 24/30 Write block 59 len 24576 data: 2 2 2 2 2 2 25/30 Read block 447 len 12288 26/30 Read block 543 len 36864 27/30 Read block 500 len 4096 28/30 Read block 372 len 4096 29/30 Write block 467 len 16384 data: 3 3 3 3 30/30 Read block 331 len 40960 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 8 6 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:22.228607357Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 1 1 1 1 1 1 T T T 3 060-079 1 1 1 1 1 1 T T T 4 080-099 1 1 1 1 1 1 T T T 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 5 5 5 T T T 24 480-499 2 2 2 5 5 5 F F F 25 500-519 2 2 2 5 5 5 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 5 5 5 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 5 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 3 3 3 6 6 6 F F F 3 060-079 3 3 3 6 6 6 F F F 4 080-099 3 3 3 6 6 6 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 2 2 2 5 5 5 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 2 2 2 5 5 5 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 3 3 3 6 6 6 F F F 24 480-499 2 2 2 5 5 5 F F F 25 500-519 2 2 2 5 5 5 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 5 5 5 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 6 Begin loop 2 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.525749971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.570533987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.571195167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.57124507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.571265857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.571283914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} {"msg":"Crucible 0249782f-bc73-40c5-8330-b21910099dc3 has session id: 06b1b2f9-4eab-49e5-b811-bb040b1d2859","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.571878613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Upstairs opts: Upstairs UUID: 0249782f-bc73-40c5-8330-b21910099dc3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.571922437Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Crucible stats registered with UUID: 0249782f-bc73-40c5-8330-b21910099dc3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.571949683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604181586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604215022Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} {"msg":"0249782f-bc73-40c5-8330-b21910099dc3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604280663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604313988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604346744Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604369651Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604400277Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604431072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604459968Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604487825Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60451707Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604547286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_connection connected from Ok(127.0.0.1:41429)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604574902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_connection connected from Ok(127.0.0.1:38672)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604608288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_connection connected from Ok(127.0.0.1:51775)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.604641353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.605224464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.605260359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.605278126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.605294894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.605311082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60532709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.605825902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60591048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606012586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606511598Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606545703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60657062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606588397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606602005Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606615294Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606630832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60664461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[0]R flush_numbers[0..12]: [3, 3, 6, 6, 6, 1, 5, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606657958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[0]R generation[0..12]: [2, 2, 3, 3, 3, 1, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606671106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606684694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[1]R flush_numbers[0..12]: [3, 3, 6, 6, 6, 1, 5, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606697922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[1]R generation[0..12]: [2, 2, 3, 3, 3, 1, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60671091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606735437Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [3, 3, 6, 6, 6, 1, 5, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606765943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[2]R generation[0..12]: [2, 2, 3, 3, 3, 1, 2, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60678747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606802098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606814896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606827674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Next flush: 7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606840093Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606852541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"":"downstairs","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606867619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606879387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"0","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606892536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"1","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606905324Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"client":"2","":"downstairs","session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606918252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.60693022Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"0249782f-bc73-40c5-8330-b21910099dc3 is now active with session: 06b1b2f9-4eab-49e5-b811-bb040b1d2859","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:05:26.606942709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606957447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198,"session_id":"06b1b2f9-4eab-49e5-b811-bb040b1d2859"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:26.606969505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 24576 data: 2 2 2 2 2 2 02/30 Write block 226 len 24576 data: 2 2 2 2 2 2 03/30 Write block 474 len 28672 data: 2 2 2 2 2 2 2 04/30 Write block 264 len 36864 data: 2 2 2 2 2 2 2 2 2 05/30 Write block 297 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Read block 182 len 16384 07/30 Flush 08/30 Write block 92 len 12288 data: 3 3 3 09/30 Write block 120 len 28672 data: 2 2 2 2 2 2 2 10/30 Write block 247 len 40960 data: 2 2 2 2 2 2 2 2 2 2 11/30 Write block 390 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Read block 121 len 36864 13/30 Write block 345 len 36864 data: 2 2 2 2 2 2 2 2 2 14/30 Write block 582 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Flush 16/30 Write block 491 len 40960 data: 2 2 2 2 2 2 2 2 3 3 17/30 Read block 466 len 12288 18/30 Flush 19/30 Write block 387 len 36864 data: 2 2 2 3 3 3 3 3 3 20/30 Write block 347 len 40960 data: 3 3 3 3 3 3 3 2 2 2 21/30 Read block 319 len 24576 22/30 Flush 23/30 Write block 343 len 20480 data: 2 2 3 3 4 24/30 Read block 266 len 40960 25/30 Flush 26/30 Write block 208 len 4096 data: 2 27/30 Read block 554 len 40960 28/30 Read block 555 len 40960 29/30 Write block 138 len 12288 data: 2 2 2 30/30 Write block 119 len 20480 data: 2 3 3 3 3 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 1 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:26.626579844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 3 3 3 6 6 6 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 1 1 1 1 1 1 T T T 6 120-139 4 4 4 8 8 8 T T T 7 140-159 1 1 1 1 1 1 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 4 4 4 7 7 7 F F F 12 240-259 4 4 4 8 8 8 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 11 11 11 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 4 4 4 10 10 10 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 5 5 5 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 4, Max flush: 11 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 2 2 2 3 3 3 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 3 3 3 6 6 6 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 5 5 5 12 12 12 F F F 6 120-139 5 5 5 12 12 12 F F F 7 140-159 5 5 5 12 12 12 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 5 5 5 12 12 12 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 4 4 4 8 8 8 F F F 13 260-279 4 4 4 7 7 7 F F F 14 280-299 4 4 4 7 7 7 F F F 15 300-319 4 4 4 7 7 7 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 11 11 11 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 4 4 4 10 10 10 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 1 1 1 1 1 1 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 4 4 4 9 9 9 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 5 5 5 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 5, Max flush: 12 Begin loop 3 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.898007313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.944447454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.945217388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.945260003Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.94527591Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.945293028Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} {"msg":"Crucible e7ed0a4a-29dd-4f26-bab7-fd04da896e1b has session id: 4189076f-7073-4a8e-86c6-2249547ba4be","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.945956747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Upstairs opts: Upstairs UUID: e7ed0a4a-29dd-4f26-bab7-fd04da896e1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.946021079Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Crucible stats registered with UUID: e7ed0a4a-29dd-4f26-bab7-fd04da896e1b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.946046505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977628017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977659833Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} {"msg":"e7ed0a4a-29dd-4f26-bab7-fd04da896e1b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977716855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977743851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977772917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977788025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977801613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977819761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977843657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977868104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977883572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.97789793Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_connection connected from Ok(127.0.0.1:65192)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.977942194Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_connection connected from Ok(127.0.0.1:43732)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978009095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_connection connected from Ok(127.0.0.1:42906)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978054868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978747164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978781139Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978808076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978834802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978863278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.978887135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.979473545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.97950693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.979534856Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980048486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980077092Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.98009149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980104088Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980117406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980130385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980142283Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980155111Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[0]R flush_numbers[0..12]: [3, 3, 6, 7, 8, 12, 12, 12, 1, 1, 12, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.98016849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[0]R generation[0..12]: [2, 2, 3, 4, 4, 5, 5, 5, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980181858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980195426Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[1]R flush_numbers[0..12]: [3, 3, 6, 7, 8, 12, 12, 12, 1, 1, 12, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980211604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[1]R generation[0..12]: [2, 2, 3, 4, 4, 5, 5, 5, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980224892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.98023834Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[2]R flush_numbers[0..12]: [3, 3, 6, 7, 8, 12, 12, 12, 1, 1, 12, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980251538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[2]R generation[0..12]: [2, 2, 3, 4, 4, 5, 5, 5, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980264596Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980278044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980294012Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980327418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980344885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980367882Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"":"downstairs","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980392989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980415836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"0","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980441732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"1","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980469298Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"client":"2","":"downstairs","session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980500184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:05:30.98052633Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"e7ed0a4a-29dd-4f26-bab7-fd04da896e1b is now active with session: 4189076f-7073-4a8e-86c6-2249547ba4be","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980556446Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980572704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206,"session_id":"4189076f-7073-4a8e-86c6-2249547ba4be"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:30.980584853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Write block 141 len 20480 data: 2 2 2 2 2 03/30 Read block 160 len 36864 04/30 Write block 306 len 12288 data: 3 2 2 05/30 Write block 445 len 40960 data: 2 2 2 2 2 2 2 2 2 2 06/30 Write block 127 len 24576 data: 2 2 2 2 2 2 07/30 Read block 552 len 24576 08/30 Write block 42 len 4096 data: 2 09/30 Write block 292 len 32768 data: 3 3 2 2 2 3 3 3 10/30 Flush 11/30 Write block 16 len 4096 data: 3 12/30 Flush 13/30 Write block 388 len 8192 data: 3 3 14/30 Write block 516 len 16384 data: 2 2 2 2 15/30 Read block 424 len 36864 16/30 Write block 439 len 20480 data: 2 2 2 2 2 17/30 Read block 439 len 12288 18/30 Write block 444 len 40960 data: 2 3 3 3 3 3 3 3 3 3 19/30 Flush 20/30 Write block 10 len 12288 data: 2 2 2 21/30 Write block 270 len 36864 data: 3 3 3 2 2 2 2 2 2 22/30 Flush 23/30 Write block 102 len 32768 data: 2 2 2 2 2 2 2 2 24/30 Write block 504 len 4096 data: 2 25/30 Read block 544 len 16384 26/30 Read block 160 len 24576 27/30 Write block 542 len 12288 data: 2 2 3 28/30 Read block 350 len 12288 29/30 Write block 246 len 24576 data: 2 3 3 3 3 3 30/30 Read block 365 len 24576 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 6 8 8 22 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:30.999499746Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 16 16 16 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 6 6 6 13 13 13 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 5 5 5 12 12 12 T T T 6 120-139 6 6 6 13 13 13 F F F 7 140-159 6 6 6 13 13 13 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 5 5 5 12 12 12 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 4 4 4 8 8 8 T T T 13 260-279 6 6 6 16 16 16 F F F 14 280-299 6 6 6 13 13 13 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 11 11 11 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 6 6 6 15 15 15 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 6 6 6 15 15 15 F F F 22 440-459 6 6 6 15 15 15 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 6 6 6 15 15 15 T T T 26 520-539 1 1 1 1 1 1 F F F 27 540-559 2 2 2 5 5 5 T T T 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 6, Max flush: 16 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 16 16 16 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 6 6 6 13 13 13 F F F 3 060-079 4 4 4 7 7 7 F F F 4 080-099 4 4 4 8 8 8 F F F 5 100-119 7 7 7 17 17 17 F F F 6 120-139 6 6 6 13 13 13 F F F 7 140-159 6 6 6 13 13 13 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 5 5 5 12 12 12 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 6 6 6 16 16 16 F F F 14 280-299 6 6 6 13 13 13 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 11 11 11 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 6 6 6 15 15 15 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 6 6 6 15 15 15 F F F 22 440-459 6 6 6 15 15 15 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 7 7 7 17 17 17 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 7 7 7 17 17 17 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 7, Max flush: 17 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.266305548Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.311997791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.312604908Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.312651611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.312667639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.312682587Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} {"msg":"Crucible 33be2ffc-f393-48b4-ba70-8f09c4d4a0c7 has session id: 68c6ce4a-76c3-472f-9153-72019b54c4d3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.313230812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Upstairs opts: Upstairs UUID: 33be2ffc-f393-48b4-ba70-8f09c4d4a0c7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.313276386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Crucible stats registered with UUID: 33be2ffc-f393-48b4-ba70-8f09c4d4a0c7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.313304182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345340312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345382926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} {"msg":"33be2ffc-f393-48b4-ba70-8f09c4d4a0c7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345447937Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345475143Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.34550004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345517708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345531706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345544834Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345562012Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.34557694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345599357Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345634732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46456)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345670107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47005)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345702052Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:58520)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.345975885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.346564405Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.346592021Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.346653293Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.346728062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.346762318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.346790234Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347190689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347262539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347302364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347815523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.34784223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347856398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347868606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347881765Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347894563Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347906511Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347919419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[0]R flush_numbers[0..12]: [16, 3, 13, 7, 8, 17, 13, 13, 1, 1, 12, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347933327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[0]R generation[0..12]: [6, 2, 6, 4, 4, 7, 6, 6, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347947065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347960754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[1]R flush_numbers[0..12]: [16, 3, 13, 7, 8, 17, 13, 13, 1, 1, 12, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.347973972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[1]R generation[0..12]: [6, 2, 6, 4, 4, 7, 6, 6, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.34798697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348000748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[2]R flush_numbers[0..12]: [16, 3, 13, 7, 8, 17, 13, 13, 1, 1, 12, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348018876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[2]R generation[0..12]: [6, 2, 6, 4, 4, 7, 6, 6, 1, 1, 5, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348032584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348046132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.34805872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:05:35.348071879Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Next flush: 18","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348089196Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348101924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"":"downstairs","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348114723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348126371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"0","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348139129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"1","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348151897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"client":"2","":"downstairs","session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348164716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348176684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"33be2ffc-f393-48b4-ba70-8f09c4d4a0c7 is now active with session: 68c6ce4a-76c3-472f-9153-72019b54c4d3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348189123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348200811Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214,"session_id":"68c6ce4a-76c3-472f-9153-72019b54c4d3"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:35.348212689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 291 len 20480 data: 3 4 4 3 3 02/30 Write block 444 len 28672 data: 3 4 4 4 4 4 4 03/30 Read block 43 len 8192 04/30 Read block 245 len 16384 05/30 Read block 394 len 28672 06/30 Write block 59 len 36864 data: 3 3 3 3 3 3 2 2 2 07/30 Read block 537 len 32768 08/30 Read block 420 len 24576 09/30 Read block 424 len 36864 10/30 Read block 285 len 8192 11/30 Read block 521 len 4096 12/30 Read block 522 len 8192 13/30 Write block 181 len 24576 data: 2 2 2 2 2 2 14/30 Write block 199 len 16384 data: 2 2 2 2 15/30 Write block 111 len 32768 data: 2 2 2 2 2 2 2 2 16/30 Write block 89 len 16384 data: 2 3 3 4 17/30 Read block 204 len 32768 18/30 Read block 86 len 8192 19/30 Read block 564 len 12288 20/30 Read block 345 len 40960 21/30 Read block 305 len 8192 22/30 Read block 529 len 16384 23/30 Write block 142 len 20480 data: 3 3 3 3 2 24/30 Write block 369 len 40960 data: 2 2 2 2 2 2 2 2 2 2 25/30 Flush 26/30 Write block 504 len 16384 data: 3 2 2 2 27/30 Write block 423 len 4096 data: 2 28/30 Read block 506 len 16384 29/30 Read block 66 len 8192 30/30 Write block 370 len 24576 data: 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:35.367114185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 16 16 16 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 8 8 8 18 18 18 F F F 3 060-079 8 8 8 18 18 18 F F F 4 080-099 8 8 8 18 18 18 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 6 6 6 13 13 13 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 6 6 6 16 16 16 F F F 14 280-299 8 8 8 18 18 18 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 11 11 11 F F F 18 360-379 8 8 8 18 18 18 T T T 19 380-399 6 6 6 15 15 15 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 6 6 6 15 15 15 T T T 22 440-459 8 8 8 18 18 18 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 7 7 7 17 17 17 T T T 26 520-539 1 1 1 1 1 1 F F F 27 540-559 7 7 7 17 17 17 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 8, Max flush: 18 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 16 16 16 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 8 8 8 18 18 18 F F F 3 060-079 8 8 8 18 18 18 F F F 4 080-099 8 8 8 18 18 18 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 6 6 6 13 13 13 F F F 7 140-159 8 8 8 18 18 18 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 6 6 6 16 16 16 F F F 14 280-299 8 8 8 18 18 18 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 1 1 1 1 1 1 F F F 17 340-359 4 4 4 11 11 11 F F F 18 360-379 9 9 9 19 19 19 F F F 19 380-399 6 6 6 15 15 15 F F F 20 400-419 6 6 6 13 13 13 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 8 8 8 18 18 18 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 1 1 1 1 1 1 F F F 27 540-559 7 7 7 17 17 17 F F F 28 560-579 2 2 2 2 2 2 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 9, Max flush: 19 Begin loop 5 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.631515755Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.676631256Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.677291066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.67733417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.677352458Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.677366996Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} {"msg":"Crucible 3f277089-6ba2-4fd8-9ab2-05515474365a has session id: 7834a9d3-956f-4ea4-aaf9-113a9b50b9ab","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.677928569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Upstairs opts: Upstairs UUID: 3f277089-6ba2-4fd8-9ab2-05515474365a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.677974703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Crucible stats registered with UUID: 3f277089-6ba2-4fd8-9ab2-05515474365a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.678036594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.70995209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.709983616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} {"msg":"3f277089-6ba2-4fd8-9ab2-05515474365a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710057236Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710079933Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710109449Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710126766Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710143214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710159452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710185868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710213984Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.71024437Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710275756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:34875)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710308362Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:36893)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710444153Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:48660)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.710490167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.71097879Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711005556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711020624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711034142Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711165145Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711206359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711607344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.71163673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.711651568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712125253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712152779Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712169737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712225049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712254976Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712270603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712283681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.7122969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[0]R flush_numbers[0..12]: [16, 3, 18, 18, 18, 18, 13, 18, 1, 18, 18, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712310278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[0]R generation[0..12]: [6, 2, 8, 8, 8, 8, 6, 8, 1, 8, 8, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712323796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712337494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[1]R flush_numbers[0..12]: [16, 3, 18, 18, 18, 18, 13, 18, 1, 18, 18, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712350992Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[1]R generation[0..12]: [6, 2, 8, 8, 8, 8, 6, 8, 1, 8, 8, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712364261Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712377949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[2]R flush_numbers[0..12]: [16, 3, 18, 18, 18, 18, 13, 18, 1, 18, 18, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712391897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[2]R generation[0..12]: [6, 2, 8, 8, 8, 8, 6, 8, 1, 8, 8, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712405005Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712419203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712431841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712444959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712459917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"All extents match","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2025-01-11T06:05:39.712480315Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"":"downstairs","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712494083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712505771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"0","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712518679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"1","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712532378Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"client":"2","":"downstairs","session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712545496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712557674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"3f277089-6ba2-4fd8-9ab2-05515474365a is now active with session: 7834a9d3-956f-4ea4-aaf9-113a9b50b9ab","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712570132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712581771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222,"session_id":"7834a9d3-956f-4ea4-aaf9-113a9b50b9ab"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:39.712593659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 443 len 16384 data: 3 4 5 5 02/30 Read block 592 len 4096 03/30 Write block 137 len 36864 data: 3 3 3 3 3 4 4 4 4 04/30 Read block 353 len 16384 05/30 Write block 382 len 28672 data: 2 2 2 2 2 3 4 06/30 Read block 63 len 20480 07/30 Write block 336 len 24576 data: 2 2 2 2 2 2 08/30 Read block 312 len 12288 09/30 Read block 369 len 36864 10/30 Read block 434 len 4096 11/30 Read block 58 len 32768 12/30 Write block 41 len 36864 data: 2 3 2 2 2 2 2 2 2 13/30 Read block 148 len 24576 14/30 Write block 519 len 20480 data: 3 2 2 2 2 15/30 Write block 353 len 20480 data: 4 3 3 3 2 16/30 Write block 376 len 28672 data: 3 3 3 2 2 2 3 17/30 Read block 118 len 4096 18/30 Read block 325 len 12288 19/30 Read block 156 len 24576 20/30 Write block 412 len 4096 data: 3 21/30 Write block 397 len 24576 data: 3 3 2 2 2 2 22/30 Write block 578 len 8192 data: 2 2 23/30 Write block 567 len 16384 data: 2 2 2 2 24/30 Write block 276 len 20480 data: 3 3 3 2 2 25/30 Read block 398 len 16384 26/30 Read block 277 len 8192 27/30 Read block 29 len 28672 28/30 Write block 338 len 20480 data: 3 3 3 3 2 29/30 Read block 44 len 24576 30/30 Read block 91 len 20480 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 6 Done Done Done false 1007 Acked Read 3 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 5 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 29 29 30 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:39.73203969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 16 16 16 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 8 8 8 18 18 18 T T T 3 060-079 8 8 8 18 18 18 F F F 4 080-099 8 8 8 18 18 18 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 6 6 6 13 13 13 T T T 7 140-159 8 8 8 18 18 18 T T T 8 160-179 1 1 1 1 1 1 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 6 6 6 16 16 16 T T T 14 280-299 8 8 8 18 18 18 T T T 15 300-319 6 6 6 13 13 13 F F F 16 320-339 1 1 1 1 1 1 T T T 17 340-359 4 4 4 11 11 11 T T T 18 360-379 9 9 9 19 19 19 T T T 19 380-399 6 6 6 15 15 15 T T T 20 400-419 6 6 6 13 13 13 T T T 21 420-439 9 9 9 19 19 19 F F F 22 440-459 8 8 8 18 18 18 T T T 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 9 9 9 19 19 19 T T T 26 520-539 1 1 1 1 1 1 T T T 27 540-559 7 7 7 17 17 17 F F F 28 560-579 2 2 2 2 2 2 T T T 29 580-599 4 4 4 8 8 8 F F F Max gen: 9, Max flush: 19 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 6 6 6 16 16 16 F F F 1 020-039 2 2 2 3 3 3 F F F 2 040-059 10 10 10 20 20 20 F F F 3 060-079 8 8 8 18 18 18 F F F 4 080-099 8 8 8 18 18 18 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 10 10 10 20 20 20 F F F 7 140-159 10 10 10 20 20 20 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 4 4 4 7 7 7 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 10 10 10 20 20 20 F F F 14 280-299 10 10 10 20 20 20 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 10 10 10 20 20 20 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 10 10 10 20 20 20 F F F 19 380-399 10 10 10 20 20 20 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 10 10 10 20 20 20 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 F F F 25 500-519 10 10 10 20 20 20 F F F 26 520-539 10 10 10 20 20 20 F F F 27 540-559 7 7 7 17 17 17 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 10, Max flush: 20 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.05187445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:44.095433095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.095835509Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.12779071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.128362462Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.128404276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.128420634Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.128443891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} {"msg":"Crucible dac2ec49-becc-41f6-821c-70d0b63e9582 has session id: 6ec778ae-c666-47f7-8ce3-116957fb55a7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.129093712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Upstairs opts: Upstairs UUID: dac2ec49-becc-41f6-821c-70d0b63e9582, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.129149085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Crucible stats registered with UUID: dac2ec49-becc-41f6-821c-70d0b63e9582","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.129188899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.160954126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.16099679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} {"msg":"dac2ec49-becc-41f6-821c-70d0b63e9582 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.16106631Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161096136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161126132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161150599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161191593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161222539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161250285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161280201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161311247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161349822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:51750)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161386317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:53513)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161465276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:40823)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.161499231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162029709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162072533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162099959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162132075Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.16216427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162195416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162667271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162761769Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.162783026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163142426Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163179931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163211267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163320782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163344849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163358557Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163387573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163402301Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[0]R flush_numbers[0..12]: [16, 3, 20, 18, 18, 18, 20, 20, 1, 18, 18, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163415799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[0]R generation[0..12]: [6, 2, 10, 8, 8, 8, 10, 10, 1, 8, 8, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163429197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163443025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[1]R flush_numbers[0..12]: [16, 3, 20, 18, 18, 18, 20, 20, 1, 18, 18, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163456183Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[1]R generation[0..12]: [6, 2, 10, 8, 8, 8, 10, 10, 1, 8, 8, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163469482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.16348332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[2]R flush_numbers[0..12]: [16, 3, 20, 18, 18, 18, 20, 20, 1, 18, 18, 7]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163496758Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[2]R generation[0..12]: [6, 2, 10, 8, 8, 8, 10, 10, 1, 8, 8, 4]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163509876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163523554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Max found gen is 11","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163540012Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Generation requested: 12 >= found:11","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.16355344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Next flush: 21","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163566208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163578867Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"":"downstairs","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163597784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163609613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"0","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163622631Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"1","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163635499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"client":"2","":"downstairs","session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163648447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163660396Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"dac2ec49-becc-41f6-821c-70d0b63e9582 is now active with session: 6ec778ae-c666-47f7-8ce3-116957fb55a7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163672874Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163684572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230,"session_id":"6ec778ae-c666-47f7-8ce3-116957fb55a7"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:44.163696681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 381 len 20480 data: 3 4 3 3 3 02/30 Read block 526 len 20480 03/30 Read block 483 len 36864 04/30 Read block 326 len 36864 05/30 Write block 139 len 20480 data: 4 4 4 5 5 06/30 Write block 379 len 12288 data: 3 3 4 07/30 Read block 233 len 28672 08/30 Read block 456 len 40960 09/30 Read block 415 len 4096 10/30 Read block 410 len 12288 11/30 Read block 236 len 40960 12/30 Write block 290 len 32768 data: 3 4 5 5 4 4 3 4 13/30 Read block 50 len 20480 14/30 Read block 520 len 36864 15/30 Read block 445 len 40960 16/30 Flush 17/30 Write block 14 len 28672 data: 2 2 4 3 3 3 3 18/30 Read block 93 len 20480 19/30 Write block 233 len 12288 data: 2 2 2 20/30 Read block 107 len 32768 21/30 Read block 571 len 24576 22/30 Read block 385 len 24576 23/30 Read block 115 len 32768 24/30 Flush 25/30 Write block 492 len 16384 data: 3 3 3 3 26/30 Read block 116 len 8192 27/30 Read block 261 len 36864 28/30 Read block 262 len 20480 29/30 Read block 114 len 36864 30/30 Read block 571 len 8192 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:44.184957414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 10 10 10 20 20 20 F F F 3 060-079 8 8 8 18 18 18 F F F 4 080-099 8 8 8 18 18 18 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 12 12 12 21 21 21 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 10 10 10 20 20 20 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 10 10 10 20 20 20 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 12 12 12 21 21 21 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 10 10 10 20 20 20 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 4 4 4 9 9 9 T T T 25 500-519 10 10 10 20 20 20 F F F 26 520-539 10 10 10 20 20 20 F F F 27 540-559 7 7 7 17 17 17 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 12, Max flush: 22 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 12 12 12 22 22 22 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 10 10 10 20 20 20 F F F 3 060-079 8 8 8 18 18 18 F F F 4 080-099 8 8 8 18 18 18 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 12 12 12 21 21 21 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 10 10 10 20 20 20 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 6 6 6 13 13 13 F F F 16 320-339 10 10 10 20 20 20 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 12 12 12 21 21 21 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 10 10 10 20 20 20 F F F 23 460-479 4 4 4 7 7 7 F F F 24 480-499 13 13 13 23 23 23 F F F 25 500-519 10 10 10 20 20 20 F F F 26 520-539 10 10 10 20 20 20 F F F 27 540-559 7 7 7 17 17 17 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 13, Max flush: 23 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.470581644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.515648482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.516292353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.516342457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.516359774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.516377342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} {"msg":"Crucible 5eea8c2a-ff94-47e3-a294-361cbcf4c5c9 has session id: bc31882f-24b6-4c36-a8a7-45fe95167ade","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.516935696Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Upstairs opts: Upstairs UUID: 5eea8c2a-ff94-47e3-a294-361cbcf4c5c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.516981199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Crucible stats registered with UUID: 5eea8c2a-ff94-47e3-a294-361cbcf4c5c9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.517001327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548609655Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548658868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} {"msg":"5eea8c2a-ff94-47e3-a294-361cbcf4c5c9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548690484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.54871718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548760004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548782801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548811617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548832604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.54886046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548887987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548909254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548931281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_connection connected from Ok(127.0.0.1:44503)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548960477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_connection connected from Ok(127.0.0.1:45733)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.548983044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_connection connected from Ok(127.0.0.1:63085)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549179357Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549825248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549877221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549900618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549928214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549955331Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.549976448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.550685851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.550735614Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.550792336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551414451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551442818Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551458795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551471543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551484762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.55149768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551509718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551522407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[0]R flush_numbers[0..12]: [22, 22, 20, 18, 18, 18, 21, 21, 1, 18, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551539704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[0]R generation[0..12]: [12, 12, 10, 8, 8, 8, 12, 12, 1, 8, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551553592Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.55156727Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[1]R flush_numbers[0..12]: [22, 22, 20, 18, 18, 18, 21, 21, 1, 18, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551580759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[1]R generation[0..12]: [12, 12, 10, 8, 8, 8, 12, 12, 1, 8, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551593967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551607605Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[2]R flush_numbers[0..12]: [22, 22, 20, 18, 18, 18, 21, 21, 1, 18, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551620563Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[2]R generation[0..12]: [12, 12, 10, 8, 8, 8, 12, 12, 1, 8, 8, 12]","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-11T06:05:48.551634801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551656388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551670116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551683014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551695453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551707871Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"":"downstairs","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.55172074Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551735597Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"0","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551749076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"1","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551762764Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"client":"2","":"downstairs","session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551776082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.55178799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"5eea8c2a-ff94-47e3-a294-361cbcf4c5c9 is now active with session: bc31882f-24b6-4c36-a8a7-45fe95167ade","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551800569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551812177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238,"session_id":"bc31882f-24b6-4c36-a8a7-45fe95167ade"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:48.551824165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 40960 data: 4 5 6 6 5 5 4 5 4 4 02/30 Read block 80 len 32768 03/30 Write block 317 len 28672 data: 2 2 2 2 2 2 2 04/30 Write block 1 len 32768 data: 2 2 2 2 2 2 2 2 05/30 Flush 06/30 Write block 430 len 28672 data: 2 2 2 2 2 2 2 07/30 Read block 300 len 8192 08/30 Write block 40 len 36864 data: 2 3 4 3 3 3 3 3 3 09/30 Flush 10/30 Write block 543 len 20480 data: 3 4 3 3 3 11/30 Read block 310 len 20480 12/30 Read block 469 len 32768 13/30 Write block 548 len 16384 data: 2 2 2 2 14/30 Write block 324 len 40960 data: 2 2 2 2 2 2 2 2 2 2 15/30 Read block 343 len 36864 16/30 Read block 7 len 36864 17/30 Write block 474 len 8192 data: 3 3 18/30 Write block 169 len 36864 data: 2 2 2 2 2 2 2 2 2 19/30 Write block 274 len 24576 data: 3 3 4 4 4 3 20/30 Read block 402 len 40960 21/30 Read block 215 len 28672 22/30 Read block 110 len 40960 23/30 Read block 83 len 4096 24/30 Read block 115 len 4096 25/30 Read block 373 len 24576 26/30 Read block 310 len 12288 27/30 Write block 89 len 8192 data: 3 4 28/30 Read block 96 len 16384 29/30 Write block 60 len 4096 data: 4 30/30 Write block 451 len 8192 data: 4 4 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 6 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:48.571761339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 24 24 24 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 8 8 8 18 18 18 T T T 4 080-099 8 8 8 18 18 18 T T T 5 100-119 8 8 8 18 18 18 F F F 6 120-139 12 12 12 21 21 21 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 1 1 1 1 1 1 T T T 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 10 10 10 20 20 20 T T T 14 280-299 14 14 14 24 24 24 F F F 15 300-319 14 14 14 24 24 24 F F F 16 320-339 14 14 14 24 24 24 T T T 17 340-359 10 10 10 20 20 20 F F F 18 360-379 12 12 12 21 21 21 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 10 10 10 20 20 20 T T T 23 460-479 4 4 4 7 7 7 T T T 24 480-499 13 13 13 23 23 23 F F F 25 500-519 10 10 10 20 20 20 F F F 26 520-539 10 10 10 20 20 20 F F F 27 540-559 7 7 7 17 17 17 T T T 28 560-579 10 10 10 20 20 20 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 14, Max flush: 25 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 14 14 14 24 24 24 F F F 1 020-039 12 12 12 22 22 22 F F F 2 040-059 14 14 14 25 25 25 F F F 3 060-079 15 15 15 26 26 26 F F F 4 080-099 15 15 15 26 26 26 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 12 12 12 21 21 21 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 15 15 15 26 26 26 F F F 9 180-199 8 8 8 18 18 18 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 7 7 7 17 17 17 F F F 13 260-279 15 15 15 26 26 26 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 14 14 14 24 24 24 F F F 16 320-339 15 15 15 26 26 26 F F F 17 340-359 10 10 10 20 20 20 F F F 18 360-379 12 12 12 21 21 21 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 14 14 14 25 25 25 F F F 22 440-459 15 15 15 26 26 26 F F F 23 460-479 15 15 15 26 26 26 F F F 24 480-499 13 13 13 23 23 23 F F F 25 500-519 10 10 10 20 20 20 F F F 26 520-539 10 10 10 20 20 20 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 10 10 10 20 20 20 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 15, Max flush: 26 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.860307339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:52.905395974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.905807777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.937855735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.938512846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.938547451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.938561119Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.938575167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} {"msg":"Crucible df3d6b11-26b6-4be7-9ba4-93281c288c2f has session id: 6c354ea2-bf07-4115-97bf-576762dfd25e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.93920974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Upstairs opts: Upstairs UUID: df3d6b11-26b6-4be7-9ba4-93281c288c2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.939277941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Crucible stats registered with UUID: df3d6b11-26b6-4be7-9ba4-93281c288c2f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.939296219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970380168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970421923Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} {"msg":"df3d6b11-26b6-4be7-9ba4-93281c288c2f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970487854Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970529728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970580161Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970656951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970752387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970784463Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970811549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970918795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970965848Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.970998174Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42833)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971022721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_connection connected from Ok(127.0.0.1:44046)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971040858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52587)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971058436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971552898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971584933Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971600311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.9716136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971633007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.971657943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972182752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972211518Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.97227104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972790669Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972819075Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972834553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972847351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972860779Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972873787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972885736Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972903123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[0]R flush_numbers[0..12]: [24, 22, 25, 26, 26, 18, 21, 21, 26, 18, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972917701Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[0]R generation[0..12]: [14, 12, 14, 15, 15, 8, 12, 12, 15, 8, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.97293134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972945218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[1]R flush_numbers[0..12]: [24, 22, 25, 26, 26, 18, 21, 21, 26, 18, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972958816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[1]R generation[0..12]: [14, 12, 14, 15, 15, 8, 12, 12, 15, 8, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972972184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.972985822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[2]R flush_numbers[0..12]: [24, 22, 25, 26, 26, 18, 21, 21, 26, 18, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.97299913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[2]R generation[0..12]: [14, 12, 14, 15, 15, 8, 12, 12, 15, 8, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973012538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:05:52.973026316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973042774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973055712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973068311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973080979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"":"downstairs","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973096447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973109165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"0","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973122023Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"1","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973134712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"client":"2","":"downstairs","session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.97314757Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973159588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"df3d6b11-26b6-4be7-9ba4-93281c288c2f is now active with session: 6c354ea2-bf07-4115-97bf-576762dfd25e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973172127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973183925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246,"session_id":"6c354ea2-bf07-4115-97bf-576762dfd25e"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:52.973195913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 14 len 36864 data: 3 3 5 4 4 4 4 3 3 02/30 Write block 522 len 12288 data: 3 3 2 03/30 Flush 04/30 Write block 182 len 16384 data: 3 3 3 3 05/30 Write block 420 len 16384 data: 2 2 2 3 06/30 Write block 316 len 20480 data: 2 3 3 3 3 07/30 Write block 497 len 40960 data: 3 3 4 4 3 2 2 4 3 3 08/30 Write block 351 len 20480 data: 4 4 5 4 4 09/30 Write block 241 len 24576 data: 2 2 2 2 2 3 10/30 Read block 44 len 8192 11/30 Read block 365 len 4096 12/30 Flush 13/30 Write block 259 len 4096 data: 2 14/30 Write block 339 len 12288 data: 4 4 4 15/30 Write block 576 len 12288 data: 2 2 3 16/30 Read block 456 len 12288 17/30 Read block 105 len 8192 18/30 Read block 372 len 28672 19/30 Read block 430 len 16384 20/30 Read block 517 len 36864 21/30 Read block 170 len 4096 22/30 Read block 344 len 24576 23/30 Read block 95 len 28672 24/30 Write block 72 len 40960 data: 2 2 2 2 3 3 3 3 3 3 25/30 Write block 448 len 8192 data: 5 5 26/30 Read block 21 len 20480 27/30 Read block 13 len 24576 28/30 Write block 55 len 28672 data: 2 2 2 2 4 5 4 29/30 Read block 418 len 20480 30/30 Read block 226 len 40960 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 1 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:52.991762974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 14 14 14 25 25 25 T T T 3 060-079 15 15 15 26 26 26 T T T 4 080-099 15 15 15 26 26 26 T T T 5 100-119 8 8 8 18 18 18 F F F 6 120-139 12 12 12 21 21 21 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 15 15 15 26 26 26 F F F 9 180-199 16 16 16 28 28 28 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 16 16 16 28 28 28 T T T 13 260-279 15 15 15 26 26 26 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 16 16 16 28 28 28 T T T 17 340-359 16 16 16 28 28 28 T T T 18 360-379 12 12 12 21 21 21 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 15 15 15 26 26 26 T T T 23 460-479 15 15 15 26 26 26 F F F 24 480-499 16 16 16 28 28 28 F F F 25 500-519 16 16 16 28 28 28 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 10 10 10 20 20 20 T T T 29 580-599 4 4 4 8 8 8 F F F Max gen: 16, Max flush: 28 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 17 17 17 29 29 29 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 8 8 8 18 18 18 F F F 6 120-139 12 12 12 21 21 21 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 15 15 15 26 26 26 F F F 9 180-199 16 16 16 28 28 28 F F F 10 200-219 8 8 8 18 18 18 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 17 17 17 29 29 29 F F F 13 260-279 15 15 15 26 26 26 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 17 17 17 29 29 29 F F F 18 360-379 12 12 12 21 21 21 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 10 10 10 20 20 20 F F F 21 420-439 16 16 16 28 28 28 F F F 22 440-459 17 17 17 29 29 29 F F F 23 460-479 15 15 15 26 26 26 F F F 24 480-499 16 16 16 28 28 28 F F F 25 500-519 16 16 16 28 28 28 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 17 17 17 29 29 29 F F F 29 580-599 4 4 4 8 8 8 F F F Max gen: 17, Max flush: 29 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.309233699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:57.353589544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.353991579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.38697334Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.387594975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.387638059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.387655036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.387672444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} {"msg":"Crucible 1cc597cc-df16-4828-aa6c-007afcecf4b6 has session id: 585e0692-517a-40f5-b0f1-ab65e9ef5cf9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.38821861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Upstairs opts: Upstairs UUID: 1cc597cc-df16-4828-aa6c-007afcecf4b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.388273082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Crucible stats registered with UUID: 1cc597cc-df16-4828-aa6c-007afcecf4b6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.388302678Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.419998444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42003176Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} {"msg":"1cc597cc-df16-4828-aa6c-007afcecf4b6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420095571Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420123227Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420148054Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420163451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42017718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420190358Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420203576Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420216274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420229542Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420243421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55815)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420447493Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48075)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420493046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:34599)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.420594593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421288918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421319664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421335831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421349439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421362528Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421381985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.421898314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42192599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422070081Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422526299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422550485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422563623Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422575812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42258906Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422601978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422614027Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422631334Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 29, 29, 29, 18, 21, 21, 26, 28, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422645372Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[0]R generation[0..12]: [16, 16, 17, 17, 17, 8, 12, 12, 15, 16, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42265885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422672639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 29, 29, 29, 18, 21, 21, 26, 28, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422685897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[1]R generation[0..12]: [16, 16, 17, 17, 17, 8, 12, 12, 15, 16, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422699115Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422712773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 29, 29, 29, 18, 21, 21, 26, 28, 18, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422726041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[2]R generation[0..12]: [16, 16, 17, 17, 17, 8, 12, 12, 15, 16, 8, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422739179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422752608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422765026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:05:57.422777644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422799311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42281188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"":"downstairs","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422824808Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422839826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"0","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422852914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"1","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422865662Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"client":"2","":"downstairs","session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.42287847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422890469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"1cc597cc-df16-4828-aa6c-007afcecf4b6 is now active with session: 585e0692-517a-40f5-b0f1-ab65e9ef5cf9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422903117Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422914775Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254,"session_id":"585e0692-517a-40f5-b0f1-ab65e9ef5cf9"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:05:57.422926754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 24576 data: 4 4 4 4 4 3 02/30 Write block 100 len 24576 data: 2 2 3 3 3 3 03/30 Read block 408 len 28672 04/30 Read block 354 len 40960 05/30 Write block 126 len 24576 data: 3 3 3 3 3 3 06/30 Write block 268 len 16384 data: 3 3 4 4 07/30 Read block 506 len 12288 08/30 Write block 418 len 24576 data: 3 3 3 3 3 4 09/30 Read block 6 len 16384 10/30 Write block 588 len 4096 data: 3 11/30 Read block 398 len 24576 12/30 Read block 213 len 40960 13/30 Write block 366 len 40960 data: 2 2 2 3 4 4 4 4 4 4 14/30 Write block 262 len 40960 data: 2 2 3 3 3 3 4 4 5 5 15/30 Flush 16/30 Write block 365 len 28672 data: 2 3 3 3 4 5 5 17/30 Write block 592 len 4096 data: 2 18/30 Read block 358 len 24576 19/30 Write block 272 len 12288 data: 4 3 4 20/30 Read block 464 len 4096 21/30 Flush 22/30 Write block 269 len 40960 data: 5 6 6 5 4 5 4 5 5 5 23/30 Flush 24/30 Write block 197 len 8192 data: 2 2 25/30 Write block 490 len 24576 data: 2 3 4 4 4 4 26/30 Read block 429 len 16384 27/30 Read block 466 len 32768 28/30 Write block 201 len 24576 data: 3 3 2 2 2 2 29/30 Write block 493 len 4096 data: 5 30/30 Write block 163 len 16384 data: 2 2 2 2 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 6 Sent Sent Done false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 2 8 Done 4 4 5 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:05:57.44204124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 18 18 18 30 30 30 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 15 15 15 26 26 26 T T T 9 180-199 16 16 16 28 28 28 T T T 10 200-219 8 8 8 18 18 18 T T T 11 220-239 12 12 12 22 22 22 F F F 12 240-259 17 17 17 29 29 29 F F F 13 260-279 18 18 18 32 32 32 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 17 17 17 29 29 29 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 18 18 18 30 30 30 F F F 22 440-459 17 17 17 29 29 29 F F F 23 460-479 15 15 15 26 26 26 F F F 24 480-499 16 16 16 28 28 28 T T T 25 500-519 16 16 16 28 28 28 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 17 17 17 29 29 29 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 18, Max flush: 32 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 18 18 18 30 30 30 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 19 19 19 33 33 33 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 19 19 19 33 33 33 F F F 11 220-239 12 12 12 22 22 22 F F F 12 240-259 17 17 17 29 29 29 F F F 13 260-279 18 18 18 32 32 32 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 17 17 17 29 29 29 F F F 18 360-379 18 18 18 31 31 31 F F F 19 380-399 12 12 12 21 21 21 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 18 18 18 30 30 30 F F F 22 440-459 17 17 17 29 29 29 F F F 23 460-479 15 15 15 26 26 26 F F F 24 480-499 19 19 19 33 33 33 F F F 25 500-519 16 16 16 28 28 28 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 15 15 15 26 26 26 F F F 28 560-579 17 17 17 29 29 29 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 19, Max flush: 33 Begin loop 10 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.712022117Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.758064952Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.758730601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.758782974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.758801131Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.758818859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} {"msg":"Crucible 3752c9b3-26dc-4d14-9ecc-4ecb2d17fde9 has session id: 8d80ed93-2d12-47b0-bc4e-011daff51944","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.759316421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Upstairs opts: Upstairs UUID: 3752c9b3-26dc-4d14-9ecc-4ecb2d17fde9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.759360545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Crucible stats registered with UUID: 3752c9b3-26dc-4d14-9ecc-4ecb2d17fde9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.759384421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79150098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791541594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} {"msg":"3752c9b3-26dc-4d14-9ecc-4ecb2d17fde9 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79157611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791593987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791619374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791641311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791658278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791675756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791692444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79171958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791749476Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791779712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_connection connected from Ok(127.0.0.1:48043)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791810187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_connection connected from Ok(127.0.0.1:54650)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791841773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_connection connected from Ok(127.0.0.1:42577)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.791873299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79237531Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.792413145Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.792507982Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.7925272Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.792791683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.792866183Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.792910067Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793076185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793384942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793496787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793519974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793534252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793590964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793609992Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79362319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793861737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793885154Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 30, 29, 29, 30, 30, 21, 33, 33, 33, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793899252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[0]R generation[0..12]: [16, 16, 18, 17, 17, 18, 18, 12, 19, 19, 19, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79391404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793928328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 30, 29, 29, 30, 30, 21, 33, 33, 33, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793946115Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[1]R generation[0..12]: [16, 16, 18, 17, 17, 18, 18, 12, 19, 19, 19, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793960054Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.793973792Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 30, 29, 29, 30, 30, 21, 33, 33, 33, 22]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79398703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[2]R generation[0..12]: [16, 16, 18, 17, 17, 18, 18, 12, 19, 19, 19, 12]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794000268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794013896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794026544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794039383Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Next flush: 34","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-11T06:06:01.79405839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794071169Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"":"downstairs","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794083927Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794095865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"0","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794108813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"1","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794121591Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"client":"2","":"downstairs","session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.79413451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794148088Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"3752c9b3-26dc-4d14-9ecc-4ecb2d17fde9 is now active with session: 8d80ed93-2d12-47b0-bc4e-011daff51944","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794160836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794172675Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262,"session_id":"8d80ed93-2d12-47b0-bc4e-011daff51944"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:01.794184573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 8192 data: 5 5 02/30 Write block 360 len 40960 data: 2 2 2 2 2 3 4 4 4 5 03/30 Read block 260 len 36864 04/30 Read block 199 len 20480 05/30 Read block 216 len 24576 06/30 Write block 540 len 8192 data: 2 2 07/30 Write block 129 len 20480 data: 4 4 4 3 2 08/30 Read block 38 len 28672 09/30 Read block 316 len 12288 10/30 Write block 316 len 16384 data: 3 4 4 4 11/30 Read block 29 len 8192 12/30 Read block 127 len 4096 13/30 Write block 230 len 12288 data: 3 3 2 14/30 Read block 512 len 24576 15/30 Read block 557 len 8192 16/30 Flush 17/30 Write block 377 len 28672 data: 4 4 4 4 5 5 4 18/30 Write block 576 len 40960 data: 3 3 4 3 2 2 3 3 3 3 19/30 Read block 54 len 4096 20/30 Read block 233 len 36864 21/30 Write block 360 len 28672 data: 3 3 3 3 3 4 5 22/30 Write block 43 len 12288 data: 4 5 5 23/30 Read block 304 len 4096 24/30 Read block 576 len 4096 25/30 Read block 527 len 24576 26/30 Write block 477 len 40960 data: 3 3 3 3 2 2 2 2 2 2 27/30 Write block 386 len 40960 data: 3 4 5 4 4 4 4 4 4 4 28/30 Read block 234 len 36864 29/30 Read block 85 len 20480 30/30 Write block 434 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 7 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:01.813652721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 18 18 18 30 30 30 T T T 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 19 19 19 33 33 33 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 19 19 19 33 33 33 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 17 17 17 29 29 29 F F F 13 260-279 18 18 18 32 32 32 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 20 20 20 34 34 34 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 20 20 20 34 34 34 F F F 18 360-379 20 20 20 34 34 34 T T T 19 380-399 12 12 12 21 21 21 T T T 20 400-419 18 18 18 30 30 30 F F F 21 420-439 18 18 18 30 30 30 T T T 22 440-459 17 17 17 29 29 29 F F F 23 460-479 15 15 15 26 26 26 T T T 24 480-499 19 19 19 33 33 33 T T T 25 500-519 16 16 16 28 28 28 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 20 20 20 34 34 34 F F F 28 560-579 17 17 17 29 29 29 T T T 29 580-599 18 18 18 31 31 31 T T T Max gen: 20, Max flush: 34 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 21 21 21 35 35 35 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 18 18 30 30 30 F F F 6 120-139 20 20 20 34 34 34 F F F 7 140-159 12 12 12 21 21 21 F F F 8 160-179 19 19 19 33 33 33 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 19 19 19 33 33 33 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 17 17 17 29 29 29 F F F 13 260-279 18 18 18 32 32 32 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 20 20 20 34 34 34 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 20 20 20 34 34 34 F F F 18 360-379 21 21 21 35 35 35 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 21 21 21 35 35 35 F F F 22 440-459 17 17 17 29 29 29 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 16 16 16 28 28 28 F F F 26 520-539 16 16 16 27 27 27 F F F 27 540-559 20 20 20 34 34 34 F F F 28 560-579 21 21 21 35 35 35 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 21, Max flush: 35 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.105142508Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.149943732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.150573306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.15061515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.150633428Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.150647896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} {"msg":"Crucible 41522d00-2b6b-4057-a572-f47faa70d1ef has session id: d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.151216418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Upstairs opts: Upstairs UUID: 41522d00-2b6b-4057-a572-f47faa70d1ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.151259972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Crucible stats registered with UUID: 41522d00-2b6b-4057-a572-f47faa70d1ef","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.151283419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182513569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182549384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} {"msg":"41522d00-2b6b-4057-a572-f47faa70d1ef active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182606466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182634162Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182658109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182673507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182687155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182700203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182717151Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182730439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182745907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182769584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:51800)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182813578Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38282)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.182841134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43854)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.18286649Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.183653623Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.183681509Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.183696157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.183709645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.183722483Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.183735631Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184234963Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184263649Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184465212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184713388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184735735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184749033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184803996Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184822773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184835601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184949585Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184968553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 35, 29, 29, 30, 34, 21, 33, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184982451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[0]R generation[0..12]: [16, 16, 21, 17, 17, 18, 20, 12, 19, 19, 19, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.184996019Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185009957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 35, 29, 29, 30, 34, 21, 33, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185023306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[1]R generation[0..12]: [16, 16, 21, 17, 17, 18, 20, 12, 19, 19, 19, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185036574Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185050252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 35, 29, 29, 30, 34, 21, 33, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185066629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[2]R generation[0..12]: [16, 16, 21, 17, 17, 18, 20, 12, 19, 19, 19, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185080208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185093876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185106484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185119322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185132071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:06:06.185144609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"":"downstairs","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185163056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185174735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"0","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185187593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"1","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185200251Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"client":"2","":"downstairs","session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185213089Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185224978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"41522d00-2b6b-4057-a572-f47faa70d1ef is now active with session: d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185237416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185249184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270,"session_id":"d5d0c9a7-2fd7-442b-a3e6-54cd3d4659e4"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:06.185262923Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 45 len 8192 data: 6 5 02/30 Write block 299 len 8192 data: 5 3 03/30 Read block 401 len 32768 04/30 Read block 552 len 20480 05/30 Write block 359 len 40960 data: 2 4 4 4 4 4 5 6 5 5 06/30 Read block 397 len 16384 07/30 Flush 08/30 Write block 333 len 4096 data: 3 09/30 Write block 501 len 40960 data: 4 3 3 5 4 4 3 2 2 2 10/30 Write block 200 len 36864 data: 3 4 4 3 3 3 3 2 3 11/30 Write block 271 len 8192 data: 7 6 12/30 Write block 382 len 40960 data: 6 5 4 4 4 5 6 5 5 5 13/30 Read block 190 len 24576 14/30 Read block 128 len 4096 15/30 Read block 410 len 16384 16/30 Read block 158 len 8192 17/30 Read block 224 len 40960 18/30 Write block 144 len 28672 data: 5 5 3 2 2 2 2 19/30 Write block 125 len 20480 data: 3 4 4 4 5 20/30 Flush 21/30 Write block 521 len 40960 data: 3 4 4 3 2 2 2 2 2 2 22/30 Read block 57 len 28672 23/30 Read block 205 len 8192 24/30 Read block 502 len 20480 25/30 Read block 378 len 8192 26/30 Write block 247 len 28672 data: 4 4 4 4 4 3 3 27/30 Write block 104 len 4096 data: 4 28/30 Read block 265 len 16384 29/30 Write block 137 len 8192 data: 4 4 30/30 Read block 423 len 4096 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:06.205220164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 18 18 30 30 30 T T T 6 120-139 22 22 22 37 37 37 T T T 7 140-159 22 22 22 37 37 37 F F F 8 160-179 19 19 19 33 33 33 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 17 17 17 29 29 29 T T T 13 260-279 22 22 22 37 37 37 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 22 22 22 36 36 36 F F F 18 360-379 22 22 22 36 36 36 F F F 19 380-399 22 22 22 37 37 37 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 21 21 21 35 35 35 F F F 22 440-459 17 17 17 29 29 29 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 16 16 16 27 27 27 T T T 27 540-559 20 20 20 34 34 34 F F F 28 560-579 21 21 21 35 35 35 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 22, Max flush: 37 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 16 16 16 27 27 27 F F F 2 040-059 22 22 22 36 36 36 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 F F F 7 140-159 22 22 22 37 37 37 F F F 8 160-179 19 19 19 33 33 33 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 23 23 23 38 38 38 F F F 13 260-279 22 22 22 37 37 37 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 22 22 22 37 37 37 F F F 17 340-359 22 22 22 36 36 36 F F F 18 360-379 22 22 22 36 36 36 F F F 19 380-399 22 22 22 37 37 37 F F F 20 400-419 18 18 18 30 30 30 F F F 21 420-439 21 21 21 35 35 35 F F F 22 440-459 17 17 17 29 29 29 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 23 23 23 38 38 38 F F F 27 540-559 20 20 20 34 34 34 F F F 28 560-579 21 21 21 35 35 35 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 23, Max flush: 38 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.490452327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.536346622Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537133604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537175758Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537191136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537208744Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} {"msg":"Crucible 4733d10c-b6d5-4265-b57a-1146c8f61bec has session id: a284dbc5-0b7f-42cd-9de8-a7f086d525e9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537806342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Upstairs opts: Upstairs UUID: 4733d10c-b6d5-4265-b57a-1146c8f61bec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537851036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Crucible stats registered with UUID: 4733d10c-b6d5-4265-b57a-1146c8f61bec","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.537876123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.569767802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.569801288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} {"msg":"4733d10c-b6d5-4265-b57a-1146c8f61bec active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.569874238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.569896784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.569934369Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.569967105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570032186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570061962Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570081679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570105656Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570136872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570172127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:53699)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570276403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45100)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570316917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45815)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570347093Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570867212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570894018Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570908246Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570921404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570935513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.570977517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.571536011Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.571564696Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.571595472Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572155846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572183812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572201409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572213968Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572227186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572240274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572252482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572265611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[0]R flush_numbers[0..12]: [27, 27, 36, 29, 29, 38, 38, 37, 33, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572278969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[0]R generation[0..12]: [16, 16, 22, 17, 17, 23, 23, 22, 19, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572292187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572305795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[1]R flush_numbers[0..12]: [27, 27, 36, 29, 29, 38, 38, 37, 33, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572319133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[1]R generation[0..12]: [16, 16, 22, 17, 17, 23, 23, 22, 19, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572332251Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.5723459Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[2]R flush_numbers[0..12]: [27, 27, 36, 29, 29, 38, 38, 37, 33, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572359148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[2]R generation[0..12]: [16, 16, 22, 17, 17, 23, 23, 22, 19, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572372356Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572385844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572400112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572420679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572434308Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572447066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"":"downstairs","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572459954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572471863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"0","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572484691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"1","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572497519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"client":"2","":"downstairs","session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572510327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572522265Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"4733d10c-b6d5-4265-b57a-1146c8f61bec is now active with session: a284dbc5-0b7f-42cd-9de8-a7f086d525e9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572534694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572546512Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278,"session_id":"a284dbc5-0b7f-42cd-9de8-a7f086d525e9"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:10.572558431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 16384 data: 2 2 2 2 02/30 Read block 279 len 24576 03/30 Read block 489 len 20480 04/30 Write block 443 len 40960 data: 4 5 6 6 5 6 6 5 5 5 05/30 Write block 428 len 16384 data: 2 2 3 3 06/30 Read block 238 len 24576 07/30 Read block 429 len 32768 08/30 Read block 267 len 32768 09/30 Read block 175 len 4096 10/30 Write block 405 len 28672 data: 2 2 2 2 2 2 3 11/30 Read block 259 len 4096 12/30 Read block 516 len 12288 13/30 Read block 210 len 16384 14/30 Read block 494 len 28672 15/30 Write block 84 len 12288 data: 2 2 2 16/30 Write block 358 len 28672 data: 2 3 5 5 5 5 5 17/30 Write block 335 len 24576 data: 2 3 3 4 5 6 18/30 Write block 164 len 28672 data: 3 3 3 2 2 3 3 19/30 Read block 84 len 20480 20/30 Flush 21/30 Write block 9 len 20480 data: 2 3 3 3 2 22/30 Read block 297 len 8192 23/30 Write block 386 len 16384 data: 5 6 7 6 24/30 Write block 42 len 28672 data: 5 5 6 7 6 5 5 25/30 Read block 534 len 32768 26/30 Write block 27 len 16384 data: 3 3 3 3 27/30 Read block 23 len 20480 28/30 Read block 416 len 24576 29/30 Write block 264 len 40960 data: 4 4 4 4 5 6 7 8 7 5 30/30 Write block 562 len 28672 data: 2 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:10.592375661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 T T T 1 020-039 16 16 16 27 27 27 T T T 2 040-059 22 22 22 36 36 36 T T T 3 060-079 17 17 17 29 29 29 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 23 23 23 38 38 38 F F F 13 260-279 22 22 22 37 37 37 T T T 14 280-299 22 22 22 36 36 36 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 24 24 24 39 39 39 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 39 39 39 F F F 19 380-399 22 22 22 37 37 37 T T T 20 400-419 24 24 24 39 39 39 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 24 24 24 39 39 39 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 23 23 23 38 38 38 F F F 27 540-559 20 20 20 34 34 34 F F F 28 560-579 21 21 21 35 35 35 T T T 29 580-599 21 21 21 35 35 35 F F F Max gen: 24, Max flush: 39 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 40 40 40 F F F 1 020-039 25 25 25 40 40 40 F F F 2 040-059 25 25 25 40 40 40 F F F 3 060-079 17 17 17 29 29 29 F F F 4 080-099 24 24 24 39 39 39 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 23 23 23 38 38 38 F F F 13 260-279 25 25 25 40 40 40 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 22 22 22 36 36 36 F F F 16 320-339 24 24 24 39 39 39 F F F 17 340-359 24 24 24 39 39 39 F F F 18 360-379 24 24 24 39 39 39 F F F 19 380-399 25 25 25 40 40 40 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 24 24 24 39 39 39 F F F 22 440-459 24 24 24 39 39 39 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 23 23 23 38 38 38 F F F 27 540-559 20 20 20 34 34 34 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 25, Max flush: 40 Begin loop 13 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.872573681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.917694701Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.918317686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.918358791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.918373079Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.918387497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} {"msg":"Crucible e7297870-a40c-4f4d-9b1a-0ebe2986cf4a has session id: 015d759d-ad10-4ba7-8780-ec0ddb954fcb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.918978466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Upstairs opts: Upstairs UUID: e7297870-a40c-4f4d-9b1a-0ebe2986cf4a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.919086011Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Crucible stats registered with UUID: e7297870-a40c-4f4d-9b1a-0ebe2986cf4a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.919119797Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950422227Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950454472Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} {"msg":"e7297870-a40c-4f4d-9b1a-0ebe2986cf4a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950504435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950524953Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950554408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950584924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.95061605Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950652735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.95068699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950718676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950791076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950832081Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33917)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950865996Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:48353)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.950936666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55503)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.951005657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.951541924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.951580858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.951610494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.95163891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.951666327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.951685784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952156349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952196134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.95222592Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952766946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952803121Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952819479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952832077Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952845675Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952858724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952874431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952887849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[0]R flush_numbers[0..12]: [40, 40, 40, 29, 39, 38, 38, 39, 39, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952901868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[0]R generation[0..12]: [25, 25, 25, 17, 24, 23, 23, 24, 24, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952915596Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952937113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[1]R flush_numbers[0..12]: [40, 40, 40, 29, 39, 38, 38, 39, 39, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952962409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[1]R generation[0..12]: [25, 25, 25, 17, 24, 23, 23, 24, 24, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.952989306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953016572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [40, 40, 40, 29, 39, 38, 38, 39, 39, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953047078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[2]R generation[0..12]: [25, 25, 25, 17, 24, 23, 23, 24, 24, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953074054Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953097261Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953121278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953145884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Next flush: 41","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953169471Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953193478Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"":"downstairs","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953210715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953222684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"0","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953235552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"1","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.95325078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"client":"2","":"downstairs","session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953274247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953294254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"e7297870-a40c-4f4d-9b1a-0ebe2986cf4a is now active with session: 015d759d-ad10-4ba7-8780-ec0ddb954fcb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953307782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953319761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286,"session_id":"015d759d-ad10-4ba7-8780-ec0ddb954fcb"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:14.953331889Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 254 len 4096 data: 3 02/30 Read block 349 len 24576 03/30 Read block 29 len 28672 04/30 Flush 05/30 Write block 325 len 20480 data: 3 3 3 3 3 06/30 Write block 453 len 24576 data: 4 3 2 2 2 2 07/30 Read block 126 len 20480 08/30 Read block 503 len 12288 09/30 Flush 10/30 Write block 318 len 16384 data: 5 5 4 3 11/30 Flush 12/30 Write block 541 len 4096 data: 3 13/30 Read block 321 len 40960 14/30 Write block 356 len 16384 data: 4 3 3 4 15/30 Read block 96 len 16384 16/30 Read block 563 len 16384 17/30 Write block 70 len 28672 data: 2 2 3 3 3 3 4 18/30 Read block 235 len 8192 19/30 Write block 88 len 24576 data: 2 4 5 4 5 4 20/30 Write block 433 len 16384 data: 3 4 4 4 21/30 Read block 472 len 12288 22/30 Write block 270 len 36864 data: 8 9 8 6 6 5 6 6 6 23/30 Read block 320 len 32768 24/30 Write block 28 len 4096 data: 4 25/30 Write block 428 len 4096 data: 3 26/30 Flush 27/30 Write block 362 len 24576 data: 6 6 6 6 7 6 28/30 Write block 274 len 4096 data: 7 29/30 Read block 198 len 16384 30/30 Write block 48 len 36864 data: 6 4 2 2 2 2 2 3 3 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:14.971272806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 40 40 40 F F F 1 020-039 26 26 26 44 44 44 F F F 2 040-059 25 25 25 40 40 40 T T T 3 060-079 26 26 26 44 44 44 F F F 4 080-099 26 26 26 44 44 44 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 26 26 26 41 41 41 F F F 13 260-279 26 26 26 44 44 44 T T T 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 26 26 26 43 43 43 F F F 17 340-359 26 26 26 44 44 44 F F F 18 360-379 24 24 24 39 39 39 T T T 19 380-399 25 25 25 40 40 40 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 26 26 26 44 44 44 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 23 23 23 38 38 38 F F F 27 540-559 26 26 26 44 44 44 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 26, Max flush: 44 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 40 40 40 F F F 1 020-039 26 26 26 44 44 44 F F F 2 040-059 27 27 27 45 45 45 F F F 3 060-079 26 26 26 44 44 44 F F F 4 080-099 26 26 26 44 44 44 F F F 5 100-119 23 23 23 38 38 38 F F F 6 120-139 23 23 23 38 38 38 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 19 19 19 33 33 33 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 26 26 26 41 41 41 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 26 26 26 43 43 43 F F F 17 340-359 26 26 26 44 44 44 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 25 25 25 40 40 40 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 26 26 26 44 44 44 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 23 23 23 38 38 38 F F F 27 540-559 26 26 26 44 44 44 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 27, Max flush: 45 Begin loop 14 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.235463825Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.281589538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282226791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282269165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282283703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282297911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} {"msg":"Crucible ace49cb0-cbcd-46e2-9e18-ec03c18b1f56 has session id: 9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282851805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Upstairs opts: Upstairs UUID: ace49cb0-cbcd-46e2-9e18-ec03c18b1f56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282896299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Crucible stats registered with UUID: ace49cb0-cbcd-46e2-9e18-ec03c18b1f56","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.282921336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314092484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314124879Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} {"msg":"ace49cb0-cbcd-46e2-9e18-ec03c18b1f56 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314201039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314223056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314255022Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314272749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314289607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314306035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314332541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314361157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314391033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314421059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:33849)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314450995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:37397)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314481851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47477)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.314513886Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315190184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315228878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315257165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315284881Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315311787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315331824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315816288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.315951379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316118037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316398828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316422805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316436843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316519332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316550208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316574714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.31668072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316706066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[0]R flush_numbers[0..12]: [40, 44, 45, 44, 44, 38, 38, 39, 39, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316719954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[0]R generation[0..12]: [25, 26, 27, 26, 26, 23, 23, 24, 24, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316733683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316748061Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[1]R flush_numbers[0..12]: [40, 44, 45, 44, 44, 38, 38, 39, 39, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316765368Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[1]R generation[0..12]: [25, 26, 27, 26, 26, 23, 23, 24, 24, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316778906Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316792655Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[2]R flush_numbers[0..12]: [40, 44, 45, 44, 44, 38, 38, 39, 39, 33, 37, 34]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316806043Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[2]R generation[0..12]: [25, 26, 27, 26, 26, 23, 23, 24, 24, 19, 22, 20]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316819351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316832889Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316845547Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316858355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Next flush: 46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316870804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":Wait for a query_work_queue command to finish before sending IO 1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316894941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"":"downstairs","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316919327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316942504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"0","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316967001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"1","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316982349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"client":"2","":"downstairs","session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.316997936Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.317010175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"ace49cb0-cbcd-46e2-9e18-ec03c18b1f56 is now active with session: 9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.317022823Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.317034482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294,"session_id":"9c5f4d2f-6f98-46aa-8380-7fd6bcb0c6f1"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:19.31704769Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 235 len 40960 data: 3 2 2 2 2 2 3 3 3 3 02/30 Read block 122 len 40960 03/30 Write block 112 len 16384 data: 3 3 3 3 04/30 Read block 263 len 32768 05/30 Read block 506 len 12288 06/30 Write block 44 len 8192 data: 7 8 07/30 Read block 155 len 20480 08/30 Read block 402 len 32768 09/30 Write block 127 len 24576 data: 5 5 6 5 5 4 10/30 Flush 11/30 Write block 322 len 32768 data: 3 3 3 4 4 4 4 4 12/30 Read block 347 len 20480 13/30 Write block 423 len 40960 data: 5 2 2 2 2 4 3 4 4 3 14/30 Flush 15/30 Flush 16/30 Write block 393 len 28672 data: 5 5 5 3 4 4 3 17/30 Write block 105 len 28672 data: 4 3 3 3 3 2 3 18/30 Read block 582 len 28672 19/30 Read block 416 len 8192 20/30 Read block 439 len 36864 21/30 Write block 539 len 36864 data: 2 3 4 3 4 5 4 4 4 22/30 Write block 425 len 12288 data: 3 3 3 23/30 Read block 467 len 12288 24/30 Write block 430 len 28672 data: 5 5 4 4 5 5 5 25/30 Write block 334 len 32768 data: 2 3 4 4 5 6 7 6 26/30 Write block 183 len 32768 data: 4 4 4 3 2 2 2 2 27/30 Write block 490 len 20480 data: 3 4 5 6 5 28/30 Write block 27 len 16384 data: 4 5 4 4 29/30 Read block 407 len 20480 30/30 Write block 67 len 32768 data: 3 2 2 3 3 4 4 4 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:19.337079611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 40 40 40 F F F 1 020-039 26 26 26 44 44 44 T T T 2 040-059 28 28 28 46 46 46 F F F 3 060-079 26 26 26 44 44 44 T T T 4 080-099 26 26 26 44 44 44 F F F 5 100-119 28 28 28 46 46 46 T T T 6 120-139 28 28 28 46 46 46 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 19 19 19 33 33 33 T T T 10 200-219 22 22 22 37 37 37 F F F 11 220-239 28 28 28 46 46 46 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 28 28 28 47 47 47 T T T 17 340-359 26 26 26 44 44 44 T T T 18 360-379 27 27 27 45 45 45 F F F 19 380-399 25 25 25 40 40 40 T T T 20 400-419 24 24 24 39 39 39 F F F 21 420-439 28 28 28 47 47 47 T T T 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 21 21 21 35 35 35 T T T 25 500-519 22 22 22 37 37 37 F F F 26 520-539 23 23 23 38 38 38 T T T 27 540-559 26 26 26 44 44 44 T T T 28 560-579 25 25 25 40 40 40 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 28, Max flush: 47 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 25 25 25 40 40 40 F F F 1 020-039 29 29 29 48 48 48 F F F 2 040-059 28 28 28 46 46 46 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 26 26 26 44 44 44 F F F 5 100-119 29 29 29 48 48 48 F F F 6 120-139 28 28 28 46 46 46 F F F 7 140-159 24 24 24 39 39 39 F F F 8 160-179 24 24 24 39 39 39 F F F 9 180-199 29 29 29 48 48 48 F F F 10 200-219 22 22 22 37 37 37 F F F 11 220-239 28 28 28 46 46 46 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 29 29 29 48 48 48 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 29 29 29 48 48 48 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 22 22 22 37 37 37 F F F 26 520-539 29 29 29 48 48 48 F F F 27 540-559 29 29 29 48 48 48 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 21 21 21 35 35 35 F F F Max gen: 29, Max flush: 48 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.645465077Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.690297687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691217322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691257746Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691271924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691287072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} {"msg":"Crucible 41730224-aa6a-42d8-9ddc-61907fdd10e8 has session id: bc02ae78-83b6-4e1b-9880-fa035487c0fb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691895009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Upstairs opts: Upstairs UUID: 41730224-aa6a-42d8-9ddc-61907fdd10e8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691939263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Crucible stats registered with UUID: 41730224-aa6a-42d8-9ddc-61907fdd10e8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.691958791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.722918077Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.722950723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} {"msg":"41730224-aa6a-42d8-9ddc-61907fdd10e8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72304208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723070397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723097443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72311466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723136617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723154685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723172173Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72318864Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723206618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723236184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:55954)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723268739Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:57792)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723471272Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:34369)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.723512946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724014337Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724049843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72406895Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724085978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724186314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724235657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724739808Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72479819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.724835045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725246999Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725275995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725291963Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725364063Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72538379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725397888Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725410157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725423455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[0]R flush_numbers[0..12]: [40, 48, 46, 48, 44, 48, 46, 39, 39, 48, 37, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725437043Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[0]R generation[0..12]: [25, 29, 28, 29, 26, 29, 28, 24, 24, 29, 22, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725450411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725464189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[1]R flush_numbers[0..12]: [40, 48, 46, 48, 44, 48, 46, 39, 39, 48, 37, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725477647Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[1]R generation[0..12]: [25, 29, 28, 29, 26, 29, 28, 24, 24, 29, 22, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725490955Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725507683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[2]R flush_numbers[0..12]: [40, 48, 46, 48, 44, 48, 46, 39, 39, 48, 37, 46]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725521271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[2]R generation[0..12]: [25, 29, 28, 29, 26, 29, 28, 24, 24, 29, 22, 28]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72553454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725548288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725560886Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725573584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Next flush: 49","v":0,"name":"crucible","level":30,"time"Wait for a query_work_queue command to finish before sending IO :"2025-01-11T06:06:23.725586073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725611209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"":"downstairs","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725625437Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725637975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"0","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725650794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"1","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725663582Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"client":"2","":"downstairs","session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.72567648Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725688548Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"41730224-aa6a-42d8-9ddc-61907fdd10e8 is now active with session: bc02ae78-83b6-4e1b-9880-fa035487c0fb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725700987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725714565Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302,"session_id":"bc02ae78-83b6-4e1b-9880-fa035487c0fb"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:23.725726563Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 4096 data: 3 02/30 Read block 190 len 4096 03/30 Write block 551 len 16384 data: 3 2 2 3 04/30 Write block 164 len 28672 data: 4 4 4 3 3 4 4 05/30 Write block 197 len 20480 data: 3 3 3 4 5 06/30 Read block 551 len 4096 07/30 Write block 23 len 16384 data: 2 2 2 3 08/30 Write block 527 len 32768 data: 3 3 3 3 2 2 2 2 09/30 Read block 56 len 36864 10/30 Flush 11/30 Write block 518 len 40960 data: 3 4 3 4 5 5 4 3 3 4 12/30 Read block 352 len 28672 13/30 Flush 14/30 Write block 97 len 12288 data: 2 2 2 15/30 Write block 9 len 16384 data: 3 4 4 4 16/30 Read block 370 len 16384 17/30 Read block 133 len 36864 18/30 Read block 546 len 20480 19/30 Read block 274 len 16384 20/30 Read block 566 len 16384 21/30 Write block 423 len 24576 data: 6 3 4 4 4 5 22/30 Read block 331 len 20480 23/30 Read block 87 len 4096 24/30 Write block 152 len 4096 data: 2 25/30 Read block 182 len 12288 26/30 Flush 27/30 Write block 222 len 20480 data: 2 2 2 2 3 28/30 Read block 284 len 40960 29/30 Write block 584 len 32768 data: 4 4 3 3 4 3 3 3 30/30 Write block 546 len 12288 data: 5 5 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 5 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:23.744201477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 51 51 51 F F F 1 020-039 30 30 30 49 49 49 F F F 2 040-059 28 28 28 46 46 46 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 29 29 29 48 48 48 F F F 6 120-139 28 28 28 46 46 46 F F F 7 140-159 30 30 30 51 51 51 F F F 8 160-179 30 30 30 49 49 49 F F F 9 180-199 30 30 30 49 49 49 F F F 10 200-219 30 30 30 49 49 49 F F F 11 220-239 28 28 28 46 46 46 T T T 12 240-259 28 28 28 46 46 46 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 29 29 29 48 48 48 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 30 30 30 51 51 51 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 30 30 30 50 50 50 F F F 26 520-539 30 30 30 50 50 50 F F F 27 540-559 30 30 30 49 49 49 T T T 28 560-579 25 25 25 40 40 40 F F F 29 580-599 21 21 21 35 35 35 T T T Max gen: 30, Max flush: 51 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 30 30 30 51 51 51 F F F 1 020-039 30 30 30 49 49 49 F F F 2 040-059 28 28 28 46 46 46 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 29 29 29 48 48 48 F F F 6 120-139 28 28 28 46 46 46 F F F 7 140-159 30 30 30 51 51 51 F F F 8 160-179 30 30 30 49 49 49 F F F 9 180-199 30 30 30 49 49 49 F F F 10 200-219 30 30 30 49 49 49 F F F 11 220-239 31 31 31 52 52 52 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 29 29 29 48 48 48 F F F 20 400-419 24 24 24 39 39 39 F F F 21 420-439 30 30 30 51 51 51 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 30 30 30 50 50 50 F F F 26 520-539 30 30 30 50 50 50 F F F 27 540-559 31 31 31 52 52 52 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 31, Max flush: 52 Begin loop 16 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.010386523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.056248372Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.05692463Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.056976073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.05699381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.057011878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} {"msg":"Crucible b2e9a317-6857-4eb3-8ce2-6f103626f748 has session id: a72f738a-7732-4ead-82e9-4d3d84eedf14","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.057494032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Upstairs opts: Upstairs UUID: b2e9a317-6857-4eb3-8ce2-6f103626f748, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.05757704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Crucible stats registered with UUID: b2e9a317-6857-4eb3-8ce2-6f103626f748","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.057622224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089349376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089381402Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} {"msg":"b2e9a317-6857-4eb3-8ce2-6f103626f748 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089431165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089451212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089479468Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089496306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089512774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089529031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089546109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089566586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089596942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089628728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_connection connected from Ok(127.0.0.1:51333)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089827071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_connection connected from Ok(127.0.0.1:50809)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.089873215Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_connection connected from Ok(127.0.0.1:32833)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.08990379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.090441147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.090467473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.090503448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.090520636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.090542553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.09056684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.09114786Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091232689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091253686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091634714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091675568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091705034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.09173494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091763936Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091791012Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091852454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.0918804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[0]R flush_numbers[0..12]: [51, 49, 46, 48, 51, 48, 46, 51, 49, 49, 49, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091896828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[0]R generation[0..12]: [30, 30, 28, 29, 30, 29, 28, 30, 30, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091910986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091924854Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[1]R flush_numbers[0..12]: [51, 49, 46, 48, 51, 48, 46, 51, 49, 49, 49, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091938112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[1]R generation[0..12]: [30, 30, 28, 29, 30, 29, 28, 30, 30, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.09195135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091965289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[2]R flush_numbers[0..12]: [51, 49, 46, 48, 51, 48, 46, 51, 49, 49, 49, 52]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091978577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[2]R generation[0..12]: [30, 30, 28, 29, 30, 29, 28, 30, 30, 30, 30, 31]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.091991845Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092005563Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092021371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092034539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Next flush: 53","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092047197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092059865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"":"downstairs","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092072524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:06:28.092084112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"0","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092103619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"1","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092116288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"client":"2","":"downstairs","session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092129306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092141404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"b2e9a317-6857-4eb3-8ce2-6f103626f748 is now active with session: a72f738a-7732-4ead-82e9-4d3d84eedf14","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092153823Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.092165471Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310,"session_id":"a72f738a-7732-4ead-82e9-4d3d84eedf14"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:28.0921775Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 403 len 32768 data: 2 2 3 3 3 3 3 3 02/30 Read block 401 len 16384 03/30 Flush 04/30 Write block 532 len 24576 data: 3 3 3 2 2 2 05/30 Write block 541 len 36864 data: 5 4 5 6 5 6 6 4 3 06/30 Read block 18 len 24576 07/30 Read block 7 len 16384 08/30 Read block 457 len 28672 09/30 Read block 6 len 12288 10/30 Flush 11/30 Write block 42 len 8192 data: 6 6 12/30 Write block 18 len 32768 data: 5 5 5 4 4 3 3 3 13/30 Write block 2 len 8192 data: 3 3 14/30 Read block 549 len 36864 15/30 Read block 77 len 36864 16/30 Read block 382 len 4096 17/30 Flush 18/30 Write block 192 len 16384 data: 2 2 2 2 19/30 Read block 485 len 16384 20/30 Read block 203 len 28672 21/30 Write block 126 len 36864 data: 5 6 6 7 6 6 5 3 2 22/30 Write block 168 len 24576 data: 4 5 5 3 3 3 23/30 Flush 24/30 Write block 204 len 40960 data: 4 4 4 3 4 2 2 2 2 2 25/30 Read block 122 len 8192 26/30 Write block 224 len 12288 data: 3 3 4 27/30 Write block 155 len 36864 data: 3 3 3 3 2 2 2 2 3 28/30 Write block 520 len 8192 data: 4 5 29/30 Read block 551 len 40960 30/30 Read block 566 len 36864 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 6 7 6 19 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:28.112197032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 29 29 29 48 48 48 F F F 6 120-139 32 32 32 56 56 56 F F F 7 140-159 30 30 30 51 51 51 T T T 8 160-179 32 32 32 56 56 56 T T T 9 180-199 32 32 32 56 56 56 F F F 10 200-219 30 30 30 49 49 49 T T T 11 220-239 31 31 31 52 52 52 T T T 12 240-259 28 28 28 46 46 46 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 29 29 29 48 48 48 F F F 20 400-419 32 32 32 53 53 53 F F F 21 420-439 30 30 30 51 51 51 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 30 30 30 50 50 50 F F F 26 520-539 32 32 32 54 54 54 T T T 27 540-559 32 32 32 54 54 54 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 32, Max flush: 56 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 32 32 32 55 55 55 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 29 29 29 48 48 48 F F F 6 120-139 32 32 32 56 56 56 F F F 7 140-159 33 33 33 57 57 57 F F F 8 160-179 33 33 33 57 57 57 F F F 9 180-199 32 32 32 56 56 56 F F F 10 200-219 33 33 33 57 57 57 F F F 11 220-239 33 33 33 57 57 57 F F F 12 240-259 28 28 28 46 46 46 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 22 22 22 36 36 36 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 29 29 29 48 48 48 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 29 29 29 48 48 48 F F F 20 400-419 32 32 32 53 53 53 F F F 21 420-439 30 30 30 51 51 51 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 30 30 30 50 50 50 F F F 26 520-539 33 33 33 57 57 57 F F F 27 540-559 32 32 32 54 54 54 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 33, Max flush: 57 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.390941842Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.43600833Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.436614317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.436655381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.43666977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.436684328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} {"msg":"Crucible 44ad8fd7-8c9b-49c7-b325-f56ec7c19d50 has session id: 068e7dc8-f3e1-44a9-ae85-526243d9db6f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.437149444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Upstairs opts: Upstairs UUID: 44ad8fd7-8c9b-49c7-b325-f56ec7c19d50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.437255719Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Crucible stats registered with UUID: 44ad8fd7-8c9b-49c7-b325-f56ec7c19d50","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.437287395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.468855839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.468898463Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} {"msg":"44ad8fd7-8c9b-49c7-b325-f56ec7c19d50 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.468961964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.468988611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469012787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469036604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469053082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.46906774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469081888Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469095076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469109504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.46913594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:37196)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469268412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63048)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469315576Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53377)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.469351941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470075262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470110307Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470125715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470140413Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.47016322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470189176Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470661492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470791584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.470828869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471231434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.4712567Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471270859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.4713285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471356497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471370965Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471383363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471399431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[0]R flush_numbers[0..12]: [55, 55, 55, 48, 51, 48, 56, 57, 57, 56, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471413479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[0]R generation[0..12]: [32, 32, 32, 29, 30, 29, 32, 33, 33, 32, 33, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471427067Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471440915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[1]R flush_numbers[0..12]: [55, 55, 55, 48, 51, 48, 56, 57, 57, 56, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471454263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[1]R generation[0..12]: [32, 32, 32, 29, 30, 29, 32, 33, 33, 32, 33, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471467632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.47148135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[2]R flush_numbers[0..12]: [55, 55, 55, 48, 51, 48, 56, 57, 57, 56, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471494828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[2]R generation[0..12]: [32, 32, 32, 29, 30, 29, 32, 33, 33, 32, 33, 33]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471508206Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471521834Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471534542Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471547361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471560539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471573457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"":"downstairs","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:06:32.471586415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471607113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"0","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471620071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"1","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471632999Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"client":"2","":"downstairs","session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471645827Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471657846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"44ad8fd7-8c9b-49c7-b325-f56ec7c19d50 is now active with session: 068e7dc8-f3e1-44a9-ae85-526243d9db6f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471670384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.471682112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318,"session_id":"068e7dc8-f3e1-44a9-ae85-526243d9db6f"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:32.47169409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 36864 data: 6 6 5 5 6 6 6 2 2 02/30 Flush 03/30 Write block 15 len 8192 data: 4 6 04/30 Write block 384 len 20480 data: 5 5 6 7 8 05/30 Write block 288 len 24576 data: 3 3 5 6 7 7 06/30 Write block 213 len 12288 data: 3 2 2 07/30 Write block 542 len 4096 data: 5 08/30 Write block 147 len 4096 data: 3 09/30 Write block 281 len 32768 data: 2 2 2 2 2 2 2 4 10/30 Write block 339 len 4096 data: 7 11/30 Read block 87 len 20480 12/30 Write block 416 len 24576 data: 3 3 4 4 4 4 13/30 Read block 126 len 40960 14/30 Read block 161 len 32768 15/30 Write block 102 len 4096 data: 4 16/30 Write block 12 len 32768 data: 5 3 4 5 7 5 6 6 17/30 Flush 18/30 Write block 321 len 28672 data: 4 4 4 4 5 5 5 19/30 Flush 20/30 Write block 328 len 36864 data: 5 5 3 3 3 4 3 4 5 21/30 Flush 22/30 Write block 332 len 8192 data: 4 5 23/30 Write block 234 len 32768 data: 3 4 3 3 3 3 3 4 24/30 Read block 487 len 4096 25/30 Read block 311 len 20480 26/30 Write block 223 len 12288 data: 3 4 4 27/30 Read block 546 len 8192 28/30 Read block 538 len 40960 29/30 Write block 182 len 28672 data: 4 5 5 5 4 3 3 30/30 Read block 114 len 32768 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 7 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 1 3 Done 9 7 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:32.490367547Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 34 34 34 59 59 59 F F F 6 120-139 32 32 32 56 56 56 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 33 33 33 57 57 57 F F F 9 180-199 32 32 32 56 56 56 T T T 10 200-219 34 34 34 59 59 59 F F F 11 220-239 33 33 33 57 57 57 T T T 12 240-259 28 28 28 46 46 46 T T T 13 260-279 27 27 27 45 45 45 F F F 14 280-299 34 34 34 59 59 59 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 34 34 34 61 61 61 T T T 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 34 34 34 59 59 59 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 30 30 30 50 50 50 F F F 26 520-539 33 33 33 57 57 57 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 34, Max flush: 61 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 29 29 29 48 48 48 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 34 34 34 59 59 59 F F F 6 120-139 32 32 32 56 56 56 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 33 33 33 57 57 57 F F F 9 180-199 35 35 35 62 62 62 F F F 10 200-219 34 34 34 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 35 35 35 62 62 62 F F F 13 260-279 27 27 27 45 45 45 F F F 14 280-299 34 34 34 59 59 59 F F F 15 300-319 26 26 26 43 43 43 F F F 16 320-339 35 35 35 62 62 62 F F F 17 340-359 29 29 29 48 48 48 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 34 34 34 59 59 59 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 F F F 25 500-519 30 30 30 50 50 50 F F F 26 520-539 33 33 33 57 57 57 F F F 27 540-559 34 34 34 59 59 59 F F F 28 560-579 25 25 25 40 40 40 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 35, Max flush: 62 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.765393525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.810345417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.810980211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.811028344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.811045892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.81106066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} {"msg":"Crucible 282926f0-1bac-43e0-afbc-d9a193e05f7e has session id: a1e8fae7-d07a-4425-8f62-37f1fa8ef81a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.811580149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Upstairs opts: Upstairs UUID: 282926f0-1bac-43e0-afbc-d9a193e05f7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.811664487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Crucible stats registered with UUID: 282926f0-1bac-43e0-afbc-d9a193e05f7e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.811685364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843137774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843169569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} {"msg":"282926f0-1bac-43e0-afbc-d9a193e05f7e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843227301Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843246319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843296432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.84338438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843432184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843465979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843499684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843733822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843765518Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843788095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58357)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843812201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44077)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843856235Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_connection connected from Ok(127.0.0.1:42854)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.843884192Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844275838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844315233Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844371965Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844405031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844431367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844463443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844891464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.844932708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845044973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845486983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845513229Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845527327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845597048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845616575Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845630923Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845643011Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845656Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[0]R flush_numbers[0..12]: [59, 55, 55, 48, 51, 59, 56, 59, 57, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845669967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[0]R generation[0..12]: [34, 32, 32, 29, 30, 34, 32, 34, 33, 35, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845683346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845697044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[1]R flush_numbers[0..12]: [59, 55, 55, 48, 51, 59, 56, 59, 57, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845713872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[1]R generation[0..12]: [34, 32, 32, 29, 30, 34, 32, 34, 33, 35, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.84572741Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845741318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[2]R flush_numbers[0..12]: [59, 55, 55, 48, 51, 59, 56, 59, 57, 62, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845754676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[2]R generation[0..12]: [34, 32, 32, 29, 30, 34, 32, 34, 33, 35, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845768004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845781502Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845794171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845806829Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Next flush: 63","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845819367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845832105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"":"downstairs","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845844824Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845864351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"0","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845877149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"1","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845889928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"client":"2","":"downstairs","session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845904795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845916964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"282926f0-1bac-43e0-afbc-d9a193e05f7e is now active with session: a1e8fae7-d07a-4425-8f62-37f1fa8ef81a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845929652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845941541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326,"session_id":"a1e8fae7-d07a-4425-8f62-37f1fa8ef81a"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:36.845953649Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 16384 data: 3 3 3 4 02/30 Read block 483 len 36864 03/30 Write block 65 len 36864 data: 3 3 4 3 3 4 4 5 5 04/30 Flush 05/30 Write block 180 len 16384 data: 2 3 5 6 06/30 Write block 339 len 28672 data: 8 8 7 4 4 4 5 07/30 Read block 217 len 32768 08/30 Write block 297 len 24576 data: 6 5 6 4 3 3 09/30 Read block 369 len 24576 10/30 Write block 289 len 12288 data: 4 6 7 11/30 Read block 126 len 32768 12/30 Read block 62 len 36864 13/30 Read block 14 len 4096 14/30 Write block 287 len 24576 data: 3 5 5 7 8 8 15/30 Write block 552 len 12288 data: 3 3 4 16/30 Read block 476 len 4096 17/30 Flush 18/30 Write block 264 len 12288 data: 5 5 5 19/30 Read block 391 len 4096 20/30 Write block 508 len 4096 data: 3 21/30 Write block 560 len 40960 data: 3 3 3 3 3 3 3 4 4 3 22/30 Write block 240 len 24576 data: 4 5 4 4 4 3 23/30 Write block 486 len 4096 data: 3 24/30 Read block 569 len 12288 25/30 Read block 525 len 24576 26/30 Write block 243 len 24576 data: 5 5 4 4 5 5 27/30 Read block 215 len 4096 28/30 Write block 188 len 28672 data: 4 3 3 2 3 3 3 29/30 Read block 384 len 36864 30/30 Write block 286 len 24576 data: 3 4 6 6 8 9 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:36.864733851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 36 36 36 63 63 63 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 34 34 34 59 59 59 F F F 6 120-139 32 32 32 56 56 56 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 33 33 33 57 57 57 F F F 9 180-199 36 36 36 64 64 64 T T T 10 200-219 34 34 34 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 35 35 35 62 62 62 T T T 13 260-279 27 27 27 45 45 45 T T T 14 280-299 36 36 36 64 64 64 T T T 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 36 36 36 64 64 64 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 34 34 34 59 59 59 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 29 29 29 48 48 48 T T T 25 500-519 30 30 30 50 50 50 T T T 26 520-539 33 33 33 57 57 57 F F F 27 540-559 36 36 36 64 64 64 F F F 28 560-579 25 25 25 40 40 40 T T T 29 580-599 31 31 31 52 52 52 F F F Max gen: 36, Max flush: 64 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 32 32 32 55 55 55 F F F 3 060-079 36 36 36 63 63 63 F F F 4 080-099 30 30 30 51 51 51 F F F 5 100-119 34 34 34 59 59 59 F F F 6 120-139 32 32 32 56 56 56 F F F 7 140-159 34 34 34 59 59 59 F F F 8 160-179 33 33 33 57 57 57 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 34 34 34 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 37 37 37 65 65 65 F F F 13 260-279 37 37 37 65 65 65 F F F 14 280-299 37 37 37 65 65 65 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 36 36 36 64 64 64 F F F 18 360-379 27 27 27 45 45 45 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 34 34 34 59 59 59 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 21 21 21 35 35 35 F F F 24 480-499 37 37 37 65 65 65 F F F 25 500-519 37 37 37 65 65 65 F F F 26 520-539 33 33 33 57 57 57 F F F 27 540-559 36 36 36 64 64 64 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 37, Max flush: 65 Begin loop 19 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.15314236Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:41.197062844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.197451741Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.229564101Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.230119654Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.23015519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.230168848Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.230183026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} {"msg":"Crucible 4f00c2b9-7e82-4aa0-839d-7285ffd7c68f has session id: e0cb041d-e4d2-4fa2-aef9-99731fbb97f7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.23080975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Upstairs opts: Upstairs UUID: 4f00c2b9-7e82-4aa0-839d-7285ffd7c68f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.230845855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Crucible stats registered with UUID: 4f00c2b9-7e82-4aa0-839d-7285ffd7c68f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.230862423Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262197868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262239123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} {"msg":"4f00c2b9-7e82-4aa0-839d-7285ffd7c68f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262307603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262359156Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262379074Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262393652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.26240753Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262490059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262523494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262555659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262585755Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:56193)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262616351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48585)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262645587Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_connection connected from Ok(127.0.0.1:42208)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.262678783Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.263381406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.263409083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.263423721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.263437339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.263451467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.263488372Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264178388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264207054Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264222052Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264792584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.26482155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264836328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264848666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264863084Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264876282Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264887971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264901019Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[0]R flush_numbers[0..12]: [59, 55, 55, 63, 51, 59, 56, 59, 57, 65, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264914397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[0]R generation[0..12]: [34, 32, 32, 36, 30, 34, 32, 34, 33, 37, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264928105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264943123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[1]R flush_numbers[0..12]: [59, 55, 55, 63, 51, 59, 56, 59, 57, 65, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264958881Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[1]R generation[0..12]: [34, 32, 32, 36, 30, 34, 32, 34, 33, 37, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.264986897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265015303Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[2]R flush_numbers[0..12]: [59, 55, 55, 63, 51, 59, 56, 59, 57, 65, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.26503586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"[2]R generation[0..12]: [34, 32, 32, 36, 30, 34, 32, 34, 33, 37, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265055228Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265086133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.26511169Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265127718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265141346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265162953Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"":"downstairs","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265187759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265210607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"0","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265229434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"1","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265246382Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"client":"2","":"downstairs","session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265269598Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265292475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"4f00c2b9-7e82-4aa0-839d-7285ffd7c68f is now active with session: e0cb041d-e4d2-4fa2-aef9-99731fbb97f7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265317212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.26533116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334,"session_id":"e0cb041d-e4d2-4fa2-aef9-99731fbb97f7"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:41.265348777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 47 len 24576 data: 6 7 5 3 3 3 02/30 Read block 489 len 40960 03/30 Read block 574 len 20480 04/30 Read block 324 len 20480 05/30 Read block 472 len 40960 06/30 Write block 155 len 28672 data: 4 4 4 4 3 3 3 07/30 Write block 289 len 20480 data: 7 9 10 9 8 08/30 Flush 09/30 Write block 367 len 16384 data: 7 6 6 6 10/30 Read block 167 len 8192 11/30 Write block 160 len 8192 data: 4 4 12/30 Write block 97 len 8192 data: 3 3 13/30 Read block 93 len 4096 14/30 Read block 456 len 24576 15/30 Write block 477 len 24576 data: 4 4 4 4 3 3 16/30 Flush 17/30 Write block 99 len 24576 data: 3 3 3 5 4 5 18/30 Write block 119 len 16384 data: 3 4 4 4 19/30 Read block 566 len 12288 20/30 Write block 151 len 24576 data: 2 3 2 2 5 5 21/30 Read block 191 len 20480 22/30 Read block 438 len 8192 23/30 Write block 155 len 16384 data: 6 6 5 5 24/30 Read block 408 len 24576 25/30 Read block 295 len 36864 26/30 Read block 525 len 36864 27/30 Read block 217 len 40960 28/30 Write block 515 len 16384 data: 2 3 3 4 29/30 Read block 348 len 4096 30/30 Read block 435 len 8192 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:41.284848841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 36 36 36 63 63 63 F F F 4 080-099 38 38 38 67 67 67 T T T 5 100-119 34 34 34 59 59 59 T T T 6 120-139 32 32 32 56 56 56 T T T 7 140-159 38 38 38 66 66 66 T T T 8 160-179 38 38 38 67 67 67 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 34 34 34 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 37 37 37 65 65 65 F F F 13 260-279 37 37 37 65 65 65 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 36 36 36 64 64 64 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 34 34 34 59 59 59 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 38 38 38 67 67 67 F F F 24 480-499 38 38 38 67 67 67 F F F 25 500-519 37 37 37 65 65 65 T T T 26 520-539 33 33 33 57 57 57 F F F 27 540-559 36 36 36 64 64 64 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 38, Max flush: 67 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 F F F 1 020-039 32 32 32 55 55 55 F F F 2 040-059 38 38 38 66 66 66 F F F 3 060-079 36 36 36 63 63 63 F F F 4 080-099 39 39 39 68 68 68 F F F 5 100-119 39 39 39 68 68 68 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 39 39 39 68 68 68 F F F 8 160-179 38 38 38 67 67 67 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 34 34 34 59 59 59 F F F 11 220-239 35 35 35 62 62 62 F F F 12 240-259 37 37 37 65 65 65 F F F 13 260-279 37 37 37 65 65 65 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 36 36 36 64 64 64 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 34 34 34 59 59 59 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 F F F 23 460-479 38 38 38 67 67 67 F F F 24 480-499 38 38 38 67 67 67 F F F 25 500-519 39 39 39 68 68 68 F F F 26 520-539 33 33 33 57 57 57 F F F 27 540-559 36 36 36 64 64 64 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 39, Max flush: 68 Begin loop 20 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.594174169Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.639207661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.639861342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.639902556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.639920664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.639936122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} {"msg":"Crucible 02a909f6-3fb5-4157-b88f-fbb2c8ef6bd0 has session id: fd044f0e-ba78-42be-9c68-07c246786dcb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.640581703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Upstairs opts: Upstairs UUID: 02a909f6-3fb5-4157-b88f-fbb2c8ef6bd0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.640615799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Crucible stats registered with UUID: 02a909f6-3fb5-4157-b88f-fbb2c8ef6bd0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.640641055Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672263791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672295767Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} {"msg":"02a909f6-3fb5-4157-b88f-fbb2c8ef6bd0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672370577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672392624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.67242453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672453666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672481242Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672509158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672538694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672560571Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672574619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672587747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43450)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672607305Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:60919)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672632811Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:62932)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.672659248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.673408145Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.673438881Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.673464558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.673492663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.67351813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.673544916Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674003174Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674052627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674131566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.67454078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674566907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674581245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674670842Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.67468994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674703168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674715866Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674728914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[0]R flush_numbers[0..12]: [59, 55, 66, 63, 68, 68, 68, 68, 67, 65, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674742193Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[0]R generation[0..12]: [34, 32, 38, 36, 39, 39, 39, 39, 38, 37, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674755681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674769439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[1]R flush_numbers[0..12]: [59, 55, 66, 63, 68, 68, 68, 68, 67, 65, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674782817Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[1]R generation[0..12]: [34, 32, 38, 36, 39, 39, 39, 39, 38, 37, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674796165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674809583Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[2]R flush_numbers[0..12]: [59, 55, 66, 63, 68, 68, 68, 68, 67, 65, 59, 62]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674826041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[2]R generation[0..12]: [34, 32, 38, 36, 39, 39, 39, 39, 38, 37, 34, 35]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674839589Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674853168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674865776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674878444Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Next flush: 69","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674897311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.67491018Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"":"downstairs","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674923158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674934956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"0","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674947665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"1","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674960193Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"client":"2","":"downstairs","session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674972951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.67498485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"02a909f6-3fb5-4157-b88f-fbb2c8ef6bd0 is now active with session: fd044f0e-ba78-42be-9c68-07c246786dcb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.674997388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.675009056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342,"session_id":"fd044f0e-ba78-42be-9c68-07c246786dcb"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:45.675020975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 258 len 36864 data: 2 3 2 2 3 3 6 6 6 02/30 Flush 03/30 Write block 58 len 16384 data: 3 5 6 5 04/30 Read block 35 len 4096 05/30 Write block 66 len 20480 data: 4 5 4 4 5 06/30 Write block 406 len 40960 data: 4 4 4 4 4 4 4 3 3 3 07/30 Read block 365 len 28672 08/30 Flush 09/30 Write block 535 len 4096 data: 3 10/30 Write block 47 len 40960 data: 7 8 6 4 4 4 3 3 4 4 11/30 Read block 299 len 24576 12/30 Write block 95 len 20480 data: 3 2 4 4 4 13/30 Read block 353 len 16384 14/30 Flush 15/30 Write block 466 len 4096 data: 2 16/30 Read block 505 len 16384 17/30 Write block 550 len 32768 data: 3 4 4 4 5 3 3 3 18/30 Write block 13 len 36864 data: 4 5 6 8 6 7 7 6 5 19/30 Read block 113 len 16384 20/30 Read block 322 len 16384 21/30 Read block 22 len 24576 22/30 Write block 472 len 28672 data: 2 2 4 4 3 5 5 23/30 Read block 20 len 24576 24/30 Read block 188 len 32768 25/30 Read block 376 len 12288 26/30 Write block 235 len 36864 data: 5 4 4 4 4 5 6 5 6 27/30 Write block 164 len 20480 data: 5 5 5 4 5 28/30 Read block 359 len 36864 29/30 Write block 440 len 4096 data: 3 30/30 Write block 104 len 32768 data: 6 5 4 4 4 4 3 4 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 1 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:45.695052676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 59 59 59 T T T 1 020-039 32 32 32 55 55 55 T T T 2 040-059 40 40 40 71 71 71 F F F 3 060-079 40 40 40 70 70 70 F F F 4 080-099 40 40 40 71 71 71 F F F 5 100-119 39 39 39 68 68 68 T T T 6 120-139 39 39 39 68 68 68 F F F 7 140-159 39 39 39 68 68 68 F F F 8 160-179 38 38 38 67 67 67 T T T 9 180-199 37 37 37 65 65 65 F F F 10 200-219 34 34 34 59 59 59 F F F 11 220-239 35 35 35 62 62 62 T T T 12 240-259 40 40 40 69 69 69 T T T 13 260-279 40 40 40 69 69 69 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 36 36 36 64 64 64 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 40 40 40 70 70 70 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 26 26 26 42 42 42 T T T 23 460-479 38 38 38 67 67 67 T T T 24 480-499 38 38 38 67 67 67 F F F 25 500-519 39 39 39 68 68 68 F F F 26 520-539 40 40 40 71 71 71 F F F 27 540-559 36 36 36 64 64 64 T T T 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 40, Max flush: 71 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 72 72 72 F F F 1 020-039 41 41 41 72 72 72 F F F 2 040-059 40 40 40 71 71 71 F F F 3 060-079 40 40 40 70 70 70 F F F 4 080-099 40 40 40 71 71 71 F F F 5 100-119 41 41 41 72 72 72 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 39 39 39 68 68 68 F F F 8 160-179 41 41 41 72 72 72 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 34 34 34 59 59 59 F F F 11 220-239 41 41 41 72 72 72 F F F 12 240-259 41 41 41 72 72 72 F F F 13 260-279 40 40 40 69 69 69 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 36 36 36 64 64 64 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 34 34 34 59 59 59 F F F 20 400-419 40 40 40 70 70 70 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 41 41 41 72 72 72 F F F 23 460-479 41 41 41 72 72 72 F F F 24 480-499 38 38 38 67 67 67 F F F 25 500-519 39 39 39 68 68 68 F F F 26 520-539 40 40 40 71 71 71 F F F 27 540-559 41 41 41 72 72 72 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 41, Max flush: 72 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:49.993784091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.03869619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.039298137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.039340662Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.039367498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.039383826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} {"msg":"Crucible 4f79169c-4267-4976-8598-b1a66b540c3a has session id: 7a48370f-68f1-4bcb-838f-fa4408b35b7b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.039987593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Upstairs opts: Upstairs UUID: 4f79169c-4267-4976-8598-b1a66b540c3a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.040031467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Crucible stats registered with UUID: 4f79169c-4267-4976-8598-b1a66b540c3a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.040055954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071455481Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071490226Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} {"msg":"4f79169c-4267-4976-8598-b1a66b540c3a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071553247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071581234Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.07160611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071623008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071647305Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071671671Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071691348Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071712965Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071738772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071770008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37754)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.07182688Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47565)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.071853916Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37055)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.07189629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.07248323Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.072548961Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.072626211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.072676104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.072714608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.072750994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073232588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073261714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073277702Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073786802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073814318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073831726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073894367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073914315Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073927553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073939431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073952429Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[0]R flush_numbers[0..12]: [72, 72, 71, 70, 71, 72, 68, 68, 72, 65, 59, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073965977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[0]R generation[0..12]: [41, 41, 40, 40, 40, 41, 39, 39, 41, 37, 34, 41]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073979366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.073992964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[1]R flush_numbers[0..12]: [72, 72, 71, 70, 71, 72, 68, 68, 72, 65, 59, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074006232Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[1]R generation[0..12]: [41, 41, 40, 40, 40, 41, 39, 39, 41, 37, 34, 41]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.07401971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074033578Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[2]R flush_numbers[0..12]: [72, 72, 71, 70, 71, 72, 68, 68, 72, 65, 59, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074047006Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[2]R generation[0..12]: [41, 41, 40, 40, 40, 41, 39, 39, 41, 37, 34, 41]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074060184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074076122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074089051Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074101869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074114477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074132974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"":"downstairs","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074146183Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074158141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"0","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074171279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"1","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074184258Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"client":"2","":"downstairs","session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074197346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074209324Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"4f79169c-4267-4976-8598-b1a66b540c3a is now active with session: 7a48370f-68f1-4bcb-838f-fa4408b35b7b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074221842Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074233541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350,"session_id":"7a48370f-68f1-4bcb-838f-fa4408b35b7b"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:50.074245499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 522 len 16384 data: 6 6 5 4 02/30 Flush 03/30 Write block 351 len 12288 data: 5 5 6 04/30 Write block 49 len 36864 data: 7 5 5 5 4 4 5 5 3 05/30 Read block 339 len 4096 06/30 Read block 540 len 28672 07/30 Read block 119 len 12288 08/30 Write block 11 len 16384 data: 5 6 5 6 09/30 Read block 519 len 28672 10/30 Read block 459 len 4096 11/30 Read block 39 len 12288 12/30 Read block 374 len 28672 13/30 Read block 40 len 32768 14/30 Flush 15/30 Write block 471 len 24576 data: 2 3 3 5 5 4 16/30 Flush 17/30 Write block 390 len 40960 data: 6 6 5 6 6 6 4 5 5 4 18/30 Read block 235 len 20480 19/30 Flush 20/30 Write block 272 len 4096 data: 9 21/30 Read block 347 len 40960 22/30 Read block 164 len 8192 23/30 Flush 24/30 Write block 341 len 40960 data: 8 5 5 5 6 4 5 4 4 4 25/30 Write block 143 len 36864 data: 6 6 6 4 4 3 3 3 3 26/30 Flush 27/30 Write block 219 len 4096 data: 2 28/30 Write block 140 len 4096 data: 5 29/30 Read block 218 len 36864 30/30 Write block 90 len 20480 data: 6 5 6 5 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:50.09318017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 F F F 1 020-039 41 41 41 72 72 72 F F F 2 040-059 42 42 42 74 74 74 F F F 3 060-079 40 40 40 70 70 70 F F F 4 080-099 40 40 40 71 71 71 T T T 5 100-119 41 41 41 72 72 72 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 42 42 42 78 78 78 T T T 8 160-179 41 41 41 72 72 72 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 34 34 34 59 59 59 T T T 11 220-239 41 41 41 72 72 72 F F F 12 240-259 41 41 41 72 72 72 F F F 13 260-279 42 42 42 77 77 77 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 42 42 42 78 78 78 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 40 40 40 70 70 70 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 41 41 41 72 72 72 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 38 38 38 67 67 67 F F F 25 500-519 39 39 39 68 68 68 F F F 26 520-539 42 42 42 73 73 73 F F F 27 540-559 41 41 41 72 72 72 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 42, Max flush: 78 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 F F F 1 020-039 41 41 41 72 72 72 F F F 2 040-059 42 42 42 74 74 74 F F F 3 060-079 40 40 40 70 70 70 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 41 41 41 72 72 72 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 43 43 43 79 79 79 F F F 8 160-179 41 41 41 72 72 72 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 41 41 41 72 72 72 F F F 12 240-259 41 41 41 72 72 72 F F F 13 260-279 42 42 42 77 77 77 F F F 14 280-299 38 38 38 66 66 66 F F F 15 300-319 36 36 36 64 64 64 F F F 16 320-339 36 36 36 64 64 64 F F F 17 340-359 42 42 42 78 78 78 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 40 40 40 70 70 70 F F F 21 420-439 34 34 34 59 59 59 F F F 22 440-459 41 41 41 72 72 72 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 38 38 38 67 67 67 F F F 25 500-519 39 39 39 68 68 68 F F F 26 520-539 42 42 42 73 73 73 F F F 27 540-559 41 41 41 72 72 72 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 31 31 31 52 52 52 F F F Max gen: 43, Max flush: 79 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.363395625Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.408305865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.408898853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.408948407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.408967134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.408986052Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} {"msg":"Crucible a6a29d50-90e0-40b8-a7aa-e18b68620f8c has session id: 30eaf701-02b5-47d0-9a1f-d32c0137f5d6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.40958703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Upstairs opts: Upstairs UUID: a6a29d50-90e0-40b8-a7aa-e18b68620f8c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.409626954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Crucible stats registered with UUID: a6a29d50-90e0-40b8-a7aa-e18b68620f8c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.409642352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441416967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441448653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} {"msg":"a6a29d50-90e0-40b8-a7aa-e18b68620f8c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441499916Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441520443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44154772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441564797Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441581095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441597033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441664833Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441706388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441737343Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44176753Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61882)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441798975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:38153)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441927297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_connection connected from Ok(127.0.0.1:42112)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.441974661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.442510158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.442551312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.442580468Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.442608555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.442635031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.442661737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443187375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443217381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443337185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443703725Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443731041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443745409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443844046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44388275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443907477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443929334Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44395758Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[0]R flush_numbers[0..12]: [74, 72, 74, 70, 79, 72, 68, 79, 72, 65, 79, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.443981107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[0]R generation[0..12]: [42, 41, 42, 40, 43, 41, 39, 43, 41, 37, 43, 41]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444024051Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444052127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[1]R flush_numbers[0..12]: [74, 72, 74, 70, 79, 72, 68, 79, 72, 65, 79, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444077843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[1]R generation[0..12]: [42, 41, 42, 40, 43, 41, 39, 43, 41, 37, 43, 41]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44410086Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444126837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[2]R flush_numbers[0..12]: [74, 72, 74, 70, 79, 72, 68, 79, 72, 65, 79, 72]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444154023Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":Wait for a query_work_queue command to finish before sending IO 1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[2]R generation[0..12]: [42, 41, 42, 40, 43, 41, 39, 43, 41, 37, 43, 41]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444232102Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44425375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444275467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444298293Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Next flush: 80","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444312971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.4443257Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"":"downstairs","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444338658Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444356795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"0","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444380742Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"1","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.44439796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"client":"2","":"downstairs","session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444411178Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444423386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"a6a29d50-90e0-40b8-a7aa-e18b68620f8c is now active with session: 30eaf701-02b5-47d0-9a1f-d32c0137f5d6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444435974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444452392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358,"session_id":"30eaf701-02b5-47d0-9a1f-d32c0137f5d6"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:54.444473909Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 4096 data: 5 02/30 Read block 504 len 20480 03/30 Flush 04/30 Flush 05/30 Write block 285 len 40960 data: 3 4 5 7 8 10 11 10 9 6 06/30 Write block 105 len 16384 data: 6 5 5 5 07/30 Read block 412 len 28672 08/30 Write block 319 len 28672 data: 6 5 5 5 5 5 6 09/30 Read block 408 len 36864 10/30 Write block 244 len 36864 data: 6 5 5 6 6 5 5 5 4 11/30 Write block 348 len 8192 data: 5 5 12/30 Write block 401 len 36864 data: 3 3 3 3 4 5 5 5 5 13/30 Write block 157 len 4096 data: 6 14/30 Write block 162 len 24576 data: 3 4 6 6 6 5 15/30 Read block 366 len 12288 16/30 Read block 37 len 4096 17/30 Write block 37 len 24576 data: 2 2 2 3 4 7 18/30 Read block 120 len 16384 19/30 Write block 497 len 16384 data: 4 4 5 5 20/30 Read block 311 len 28672 21/30 Read block 86 len 24576 22/30 Write block 487 len 36864 data: 2 2 2 4 5 6 7 6 5 23/30 Write block 598 len 8192 data: 2 2 24/30 Write block 234 len 36864 data: 4 6 5 5 5 5 6 7 6 25/30 Flush 26/30 Write block 151 len 24576 data: 4 4 3 3 7 7 27/30 Write block 65 len 12288 data: 4 5 6 28/30 Read block 420 len 24576 29/30 Read block 429 len 4096 30/30 Write block 220 len 28672 data: 2 2 3 4 5 5 5 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:54.463210707Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 F F F 1 020-039 44 44 44 82 82 82 F F F 2 040-059 44 44 44 82 82 82 F F F 3 060-079 40 40 40 70 70 70 T T T 4 080-099 43 43 43 79 79 79 F F F 5 100-119 44 44 44 82 82 82 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 44 44 44 82 82 82 T T T 8 160-179 44 44 44 82 82 82 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 44 44 44 82 82 82 T T T 12 240-259 44 44 44 82 82 82 F F F 13 260-279 42 42 42 77 77 77 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 44 44 44 82 82 82 F F F 16 320-339 44 44 44 82 82 82 F F F 17 340-359 44 44 44 82 82 82 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 44 44 44 82 82 82 F F F 21 420-439 44 44 44 80 80 80 F F F 22 440-459 41 41 41 72 72 72 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 44 44 44 82 82 82 F F F 25 500-519 44 44 44 82 82 82 F F F 26 520-539 42 42 42 73 73 73 F F F 27 540-559 41 41 41 72 72 72 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 44, Max flush: 82 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 74 74 74 F F F 1 020-039 44 44 44 82 82 82 F F F 2 040-059 44 44 44 82 82 82 F F F 3 060-079 45 45 45 83 83 83 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 44 44 44 82 82 82 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 45 45 45 83 83 83 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 45 45 45 83 83 83 F F F 12 240-259 44 44 44 82 82 82 F F F 13 260-279 42 42 42 77 77 77 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 44 44 44 82 82 82 F F F 16 320-339 44 44 44 82 82 82 F F F 17 340-359 44 44 44 82 82 82 F F F 18 360-379 38 38 38 67 67 67 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 44 44 44 82 82 82 F F F 21 420-439 44 44 44 80 80 80 F F F 22 440-459 41 41 41 72 72 72 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 44 44 44 82 82 82 F F F 25 500-519 44 44 44 82 82 82 F F F 26 520-539 42 42 42 73 73 73 F F F 27 540-559 41 41 41 72 72 72 F F F 28 560-579 37 37 37 65 65 65 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 45, Max flush: 83 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.728686041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.775928831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.776824198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.776868612Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.77688381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.776901498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} {"msg":"Crucible 2c887c4f-6247-4a88-95f3-1316744e0ca5 has session id: 18836308-7a4f-4aac-bb17-ea096bf0ff79","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.777445573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Upstairs opts: Upstairs UUID: 2c887c4f-6247-4a88-95f3-1316744e0ca5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.777581695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Crucible stats registered with UUID: 2c887c4f-6247-4a88-95f3-1316744e0ca5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.777610901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809383616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809415252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} {"msg":"2c887c4f-6247-4a88-95f3-1316744e0ca5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809486222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809508439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809539475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809561802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809579199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809636122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809679706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809701683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809729469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809756505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_connection connected from Ok(127.0.0.1:44166)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809787301Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_connection connected from Ok(127.0.0.1:62536)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809815347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_connection connected from Ok(127.0.0.1:45275)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.809981715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.810640325Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.810682219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.810713954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.81074316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.810769387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.810794093Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811288766Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811433826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811461322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811836911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811866987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811892533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811953695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.811974072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.81198733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812046312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.81206456Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[0]R flush_numbers[0..12]: [74, 82, 82, 83, 79, 82, 68, 83, 82, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812078128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[0]R generation[0..12]: [42, 44, 44, 45, 43, 44, 39, 45, 44, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812091596Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812105814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[1]R flush_numbers[0..12]: [74, 82, 82, 83, 79, 82, 68, 83, 82, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812123432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[1]R generation[0..12]: [42, 44, 44, 45, 43, 44, 39, 45, 44, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.81213708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812151168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[2]R flush_numbers[0..12]: [74, 82, 82, 83, 79, 82, 68, 83, 82, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812164666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[2]R generation[0..12]: [42, 44, 44, 45, 43, 44, 39, 45, 44, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812178004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812191572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812204221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812217069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812229647Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:06:58.812242335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"":"downstairs","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812260373Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812272151Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"0","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.81228489Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"1","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812297678Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"client":"2","":"downstairs","session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812310436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812324244Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"2c887c4f-6247-4a88-95f3-1316744e0ca5 is now active with session: 18836308-7a4f-4aac-bb17-ea096bf0ff79","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812336922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812348801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366,"session_id":"18836308-7a4f-4aac-bb17-ea096bf0ff79"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:06:58.812360739Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 154 len 16384 data: 4 8 8 7 02/30 Write block 18 len 12288 data: 8 8 7 03/30 Write block 523 len 24576 data: 7 6 5 4 5 4 04/30 Read block 171 len 8192 05/30 Write block 315 len 4096 data: 2 06/30 Flush 07/30 Write block 41 len 24576 data: 5 8 7 8 9 7 08/30 Write block 75 len 20480 data: 4 5 4 4 4 09/30 Write block 558 len 24576 data: 3 3 4 4 4 4 10/30 Write block 452 len 8192 data: 6 5 11/30 Write block 379 len 4096 data: 5 12/30 Read block 445 len 12288 13/30 Read block 165 len 24576 14/30 Read block 325 len 24576 15/30 Flush 16/30 Write block 509 len 4096 data: 3 17/30 Read block 358 len 4096 18/30 Write block 112 len 16384 data: 4 4 4 4 19/30 Write block 514 len 4096 data: 2 20/30 Write block 355 len 4096 data: 5 21/30 Read block 109 len 36864 22/30 Write block 254 len 28672 data: 4 3 3 2 3 4 3 23/30 Write block 415 len 28672 data: 4 4 4 5 5 6 5 24/30 Write block 421 len 12288 data: 6 4 7 25/30 Read block 90 len 28672 26/30 Write block 171 len 8192 data: 4 4 27/30 Read block 226 len 8192 28/30 Read block 172 len 28672 29/30 Read block 368 len 24576 30/30 Write block 170 len 24576 data: 6 5 5 4 3 3 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 1 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:06:58.828681378Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 84 84 84 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 46 46 46 85 85 85 F F F 3 060-079 46 46 46 85 85 85 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 44 44 44 82 82 82 T T T 6 120-139 39 39 39 68 68 68 F F F 7 140-159 46 46 46 84 84 84 F F F 8 160-179 44 44 44 82 82 82 T T T 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 45 45 45 83 83 83 F F F 12 240-259 44 44 44 82 82 82 T T T 13 260-279 42 42 42 77 77 77 T T T 14 280-299 44 44 44 82 82 82 F F F 15 300-319 46 46 46 84 84 84 F F F 16 320-339 44 44 44 82 82 82 F F F 17 340-359 44 44 44 82 82 82 T T T 18 360-379 46 46 46 85 85 85 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 44 44 44 82 82 82 T T T 21 420-439 44 44 44 80 80 80 T T T 22 440-459 46 46 46 85 85 85 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 44 44 44 82 82 82 F F F 25 500-519 44 44 44 82 82 82 T T T 26 520-539 46 46 46 84 84 84 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 46 46 46 85 85 85 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 46, Max flush: 85 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 84 84 84 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 46 46 46 85 85 85 F F F 3 060-079 46 46 46 85 85 85 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 47 47 47 86 86 86 F F F 6 120-139 39 39 39 68 68 68 F F F 7 140-159 46 46 46 84 84 84 F F F 8 160-179 47 47 47 86 86 86 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 45 45 45 83 83 83 F F F 12 240-259 47 47 47 86 86 86 F F F 13 260-279 47 47 47 86 86 86 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 46 46 46 84 84 84 F F F 16 320-339 44 44 44 82 82 82 F F F 17 340-359 47 47 47 86 86 86 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 47 47 47 86 86 86 F F F 22 440-459 46 46 46 85 85 85 F F F 23 460-479 42 42 42 75 75 75 F F F 24 480-499 44 44 44 82 82 82 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 46 46 46 84 84 84 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 46 46 46 85 85 85 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 47, Max flush: 86 Begin loop 24 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.124775355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:03.168856568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.169254424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.201919327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.202498008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.202533313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.202548271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.202562379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} {"msg":"Crucible ce45e3d3-0867-4ff0-b34a-4bcd1435301e has session id: c493a17e-82ce-4c78-ae5a-63d1e0f42650","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.203225799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Upstairs opts: Upstairs UUID: ce45e3d3-0867-4ff0-b34a-4bcd1435301e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.203269262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Crucible stats registered with UUID: ce45e3d3-0867-4ff0-b34a-4bcd1435301e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.203293499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234539107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234570803Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} {"msg":"ce45e3d3-0867-4ff0-b34a-4bcd1435301e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234633404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234667329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234698665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234717533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.23473805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234763416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234792602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234822468Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234851554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.23488291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_connection connected from Ok(127.0.0.1:46251)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234916246Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_connection connected from Ok(127.0.0.1:62576)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234948481Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_connection connected from Ok(127.0.0.1:40519)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.234981137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.235754601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.235791256Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.235819692Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.235846548Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.235874954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.235893252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.236380725Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.236454895Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.236480492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.236886946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.236914542Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.23692891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237017238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237043724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237057263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237117205Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237136492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[0]R flush_numbers[0..12]: [84, 84, 85, 85, 79, 86, 68, 84, 86, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.23715034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[0]R generation[0..12]: [46, 46, 46, 46, 43, 47, 39, 46, 47, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237164438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237178366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[1]R flush_numbers[0..12]: [84, 84, 85, 85, 79, 86, 68, 84, 86, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237191894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[1]R generation[0..12]: [46, 46, 46, 46, 43, 47, 39, 46, 47, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237208722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.23722273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[2]R flush_numbers[0..12]: [84, 84, 85, 85, 79, 86, 68, 84, 86, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237236368Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[2]R generation[0..12]: [46, 46, 46, 46, 43, 47, 39, 46, 47, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237249496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237263125Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237275713Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237288521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237301029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","Wait for a query_work_queue command to finish before sending IO pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237319677Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"":"downstairs","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237332865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237344753Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"0","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237357761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"1","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.23737079Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"client":"2","":"downstairs","session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237395706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237408915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"ce45e3d3-0867-4ff0-b34a-4bcd1435301e is now active with session: c493a17e-82ce-4c78-ae5a-63d1e0f42650","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237424552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237436441Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374,"session_id":"c493a17e-82ce-4c78-ae5a-63d1e0f42650"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:03.237448419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 428 len 24576 data: 6 4 7 7 6 6 02/30 Write block 422 len 36864 data: 5 8 4 5 5 5 7 5 8 03/30 Write block 429 len 8192 data: 6 9 04/30 Write block 534 len 12288 data: 4 4 3 05/30 Write block 129 len 32768 data: 8 7 7 6 4 3 3 3 06/30 Read block 247 len 12288 07/30 Read block 70 len 16384 08/30 Read block 295 len 8192 09/30 Flush 10/30 Write block 271 len 4096 data: 10 11/30 Flush 12/30 Write block 458 len 12288 data: 3 2 2 13/30 Flush 14/30 Write block 569 len 4096 data: 4 15/30 Read block 367 len 28672 16/30 Flush 17/30 Write block 453 len 4096 data: 6 18/30 Read block 169 len 16384 19/30 Write block 461 len 12288 data: 2 2 2 20/30 Write block 144 len 40960 data: 7 7 5 5 4 4 4 5 5 4 21/30 Flush 22/30 Write block 172 len 32768 data: 6 5 4 4 3 3 2 2 23/30 Flush 24/30 Write block 4 len 20480 data: 4 3 3 3 3 25/30 Read block 12 len 20480 26/30 Write block 313 len 40960 data: 2 2 3 4 5 6 7 6 6 6 27/30 Read block 184 len 20480 28/30 Write block 422 len 24576 data: 6 9 5 6 6 6 29/30 Read block 260 len 8192 30/30 Read block 539 len 40960 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 5 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 10 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:03.255795831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 46 46 46 84 84 84 T T T 1 020-039 46 46 46 84 84 84 F F F 2 040-059 46 46 46 85 85 85 F F F 3 060-079 46 46 46 85 85 85 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 47 47 47 86 86 86 F F F 6 120-139 48 48 48 87 87 87 F F F 7 140-159 48 48 48 91 91 91 F F F 8 160-179 48 48 48 92 92 92 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 45 45 45 83 83 83 F F F 12 240-259 47 47 47 86 86 86 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 46 46 46 84 84 84 T T T 16 320-339 44 44 44 82 82 82 T T T 17 340-359 47 47 47 86 86 86 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 48 48 48 87 87 87 T T T 22 440-459 48 48 48 91 91 91 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 44 44 44 82 82 82 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 48 48 48 87 87 87 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 48 48 48 90 90 90 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 48, Max flush: 92 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 93 93 93 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 46 46 46 85 85 85 F F F 3 060-079 46 46 46 85 85 85 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 47 47 47 86 86 86 F F F 6 120-139 48 48 48 87 87 87 F F F 7 140-159 48 48 48 91 91 91 F F F 8 160-179 48 48 48 92 92 92 F F F 9 180-199 37 37 37 65 65 65 F F F 10 200-219 43 43 43 79 79 79 F F F 11 220-239 45 45 45 83 83 83 F F F 12 240-259 47 47 47 86 86 86 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 49 49 49 93 93 93 F F F 16 320-339 49 49 49 93 93 93 F F F 17 340-359 47 47 47 86 86 86 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 49 49 49 93 93 93 F F F 22 440-459 48 48 48 91 91 91 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 44 44 44 82 82 82 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 48 48 48 87 87 87 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 48 48 48 90 90 90 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 49, Max flush: 93 Begin loop 25 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.556709378Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.601590442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.602230784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.602273838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.602293466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.602308973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} {"msg":"Crucible 3eb01a7d-031a-42cc-9b2b-f53a2b0d1c79 has session id: 507fa4c8-2cb7-47c6-84cf-37f3852b694c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.602930739Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Upstairs opts: Upstairs UUID: 3eb01a7d-031a-42cc-9b2b-f53a2b0d1c79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.602973163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Crucible stats registered with UUID: 3eb01a7d-031a-42cc-9b2b-f53a2b0d1c79","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.60299623Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635145844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635179889Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} {"msg":"3eb01a7d-031a-42cc-9b2b-f53a2b0d1c79 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635231792Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635252289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635280335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635297703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.6353141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635330049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635346496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635362754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635384231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635406918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_connection connected from Ok(127.0.0.1:38864)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635479078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_connection connected from Ok(127.0.0.1:48202)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635775457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40504)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.635818172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636244693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.63627326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636288977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636311784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636337711Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636367247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636874277Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.636904793Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637151619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637473055Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637505361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637520689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637533087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637546235Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637559274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637630354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637649551Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[0]R flush_numbers[0..12]: [93, 84, 85, 85, 79, 86, 87, 91, 92, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637663189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[0]R generation[0..12]: [49, 46, 46, 46, 43, 47, 48, 48, 48, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637677657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637691506Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[1]R flush_numbers[0..12]: [93, 84, 85, 85, 79, 86, 87, 91, 92, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637704784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[1]R generation[0..12]: [49, 46, 46, 46, 43, 47, 48, 48, 48, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.63772913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637744268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[2]R flush_numbers[0..12]: [93, 84, 85, 85, 79, 86, 87, 91, 92, 65, 79, 83]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637757597Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[2]R generation[0..12]: [49, 46, 46, 46, 43, 47, 48, 48, 48, 37, 43, 45]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637770935Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637788452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.63780148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637814289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7Wait for a query_work_queue command to finish before sending IO ","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Next flush: 94","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637833826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637846274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"":"downstairs","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637858993Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637870781Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"0","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637883599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"1","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637896198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"client":"2","":"downstairs","session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637908966Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637920824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"3eb01a7d-031a-42cc-9b2b-f53a2b0d1c79 is now active with session: 507fa4c8-2cb7-47c6-84cf-37f3852b694c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637933313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637944901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382,"session_id":"507fa4c8-2cb7-47c6-84cf-37f3852b694c"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:07.637956639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 209 len 12288 data: 3 3 3 02/30 Read block 317 len 20480 03/30 Write block 110 len 32768 data: 4 5 5 5 5 5 3 3 04/30 Read block 588 len 16384 05/30 Read block 335 len 28672 06/30 Flush 07/30 Write block 197 len 16384 data: 4 4 4 5 08/30 Write block 57 len 20480 data: 4 4 6 7 6 09/30 Write block 496 len 12288 data: 3 5 5 10/30 Read block 217 len 4096 11/30 Read block 570 len 20480 12/30 Read block 91 len 40960 13/30 Write block 13 len 12288 data: 6 7 7 14/30 Write block 256 len 12288 data: 4 3 4 15/30 Flush 16/30 Write block 236 len 8192 data: 6 6 17/30 Read block 507 len 20480 18/30 Read block 297 len 40960 19/30 Read block 335 len 32768 20/30 Read block 490 len 20480 21/30 Write block 525 len 32768 data: 6 5 6 5 4 4 3 4 22/30 Write block 141 len 12288 data: 5 6 7 23/30 Read block 506 len 20480 24/30 Read block 279 len 36864 25/30 Read block 494 len 20480 26/30 Read block 501 len 4096 27/30 Read block 193 len 16384 28/30 Write block 65 len 40960 data: 5 6 7 5 5 6 5 6 6 5 29/30 Write block 218 len 28672 data: 2 3 3 3 4 5 6 30/30 Write block 482 len 8192 data: 4 3 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 2 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:07.65637784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 50 50 50 95 95 95 F F F 3 060-079 50 50 50 95 95 95 T T T 4 080-099 43 43 43 79 79 79 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 48 48 48 87 87 87 F F F 7 140-159 48 48 48 91 91 91 T T T 8 160-179 48 48 48 92 92 92 F F F 9 180-199 50 50 50 95 95 95 F F F 10 200-219 50 50 50 95 95 95 T T T 11 220-239 45 45 45 83 83 83 T T T 12 240-259 50 50 50 95 95 95 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 49 49 49 93 93 93 F F F 16 320-339 49 49 49 93 93 93 F F F 17 340-359 47 47 47 86 86 86 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 49 49 49 93 93 93 F F F 22 440-459 48 48 48 91 91 91 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 50 50 50 95 95 95 T T T 25 500-519 47 47 47 86 86 86 F F F 26 520-539 48 48 48 87 87 87 T T T 27 540-559 46 46 46 85 85 85 F F F 28 560-579 48 48 48 90 90 90 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 50, Max flush: 95 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 50 50 50 95 95 95 F F F 3 060-079 51 51 51 96 96 96 F F F 4 080-099 43 43 43 79 79 79 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 48 48 48 87 87 87 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 48 48 48 92 92 92 F F F 9 180-199 50 50 50 95 95 95 F F F 10 200-219 51 51 51 96 96 96 F F F 11 220-239 51 51 51 96 96 96 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 44 44 44 82 82 82 F F F 15 300-319 49 49 49 93 93 93 F F F 16 320-339 49 49 49 93 93 93 F F F 17 340-359 47 47 47 86 86 86 F F F 18 360-379 46 46 46 85 85 85 F F F 19 380-399 42 42 42 76 76 76 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 49 49 49 93 93 93 F F F 22 440-459 48 48 48 91 91 91 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 51 51 51 96 96 96 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 48 48 48 90 90 90 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 51, Max flush: 96 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.936744607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.981424368Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.9820667Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.982110174Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.982125492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.98214033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} {"msg":"Crucible cb5e8600-d6ca-43ef-af39-40410cd604f7 has session id: 28e3df0f-a467-44d2-980d-9ca2f1d7a1cd","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.982688175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Upstairs opts: Upstairs UUID: cb5e8600-d6ca-43ef-af39-40410cd604f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.982732419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Crucible stats registered with UUID: cb5e8600-d6ca-43ef-af39-40410cd604f7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:11.982759515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014182299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014214664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} {"msg":"cb5e8600-d6ca-43ef-af39-40410cd604f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014284135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014306972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014336488Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014357045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014387321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014417717Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014447483Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014478858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014510164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.01454091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:55423)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014574825Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:61024)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.01461393Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_connection connected from Ok(127.0.0.1:43833)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.014748271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.015308385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.015335371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.015349969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.015363407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.015376945Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.015390044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016010629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016101926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.01614958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016569782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016601698Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016626445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016639883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016656521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016669769Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016727861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016747258Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[0]R flush_numbers[0..12]: [95, 84, 95, 96, 79, 94, 87, 96, 92, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016761456Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[0]R generation[0..12]: [50, 46, 50, 51, 43, 50, 48, 51, 48, 50, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016775194Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016788972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[1]R flush_numbers[0..12]: [95, 84, 95, 96, 79, 94, 87, 96, 92, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016802311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[1]R generation[0..12]: [50, 46, 50, 51, 43, 50, 48, 51, 48, 50, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016815559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016829087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[2]R flush_numbers[0..12]: [95, 84, 95, 96, 79, 94, 87, 96, 92, 95, 96, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016842295Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[2]R generation[0..12]: [50, 46, 50, 51, 43, 50, 48, 51, 48, 50, 51, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016855613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016869241Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.01688212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016895208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016909966Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016922954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"":"downstairs","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016935752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016963249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"0","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016977727Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"1","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.016990575Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"client":"2","":"downstairs","session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.017003513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.017015521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"cb5e8600-d6ca-43ef-af39-40410cd604f7 is now active with session: 28e3df0f-a467-44d2-980d-9ca2f1d7a1cd","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.01702799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.017039788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390,"session_id":"28e3df0f-a467-44d2-980d-9ca2f1d7a1cd"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:12.017051636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 177 len 28672 data: 4 3 3 3 4 6 7 02/30 Read block 383 len 24576 03/30 Write block 439 len 32768 data: 3 4 3 3 5 6 7 7 04/30 Write block 94 len 12288 data: 5 4 3 05/30 Read block 46 len 24576 06/30 Write block 120 len 12288 data: 5 5 5 07/30 Write block 386 len 32768 data: 7 8 9 7 7 7 6 7 08/30 Read block 155 len 4096 09/30 Write block 195 len 16384 data: 3 2 5 5 10/30 Write block 186 len 12288 data: 5 4 5 11/30 Read block 415 len 32768 12/30 Write block 283 len 36864 data: 3 3 4 5 6 8 9 11 12 13/30 Read block 54 len 20480 14/30 Read block 301 len 24576 15/30 Read block 258 len 12288 16/30 Read block 229 len 12288 17/30 Read block 228 len 40960 18/30 Write block 336 len 40960 data: 6 5 6 9 9 9 6 6 6 7 19/30 Write block 215 len 20480 data: 3 2 2 3 4 20/30 Write block 375 len 28672 data: 5 4 5 5 6 5 6 21/30 Read block 486 len 28672 22/30 Read block 528 len 36864 23/30 Read block 14 len 28672 24/30 Write block 86 len 4096 data: 3 25/30 Read block 391 len 24576 26/30 Read block 404 len 28672 27/30 Read block 206 len 28672 28/30 Read block 220 len 12288 29/30 Read block 273 len 8192 30/30 Read block 15 len 20480 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Write 8 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:12.037428401Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 50 50 50 95 95 95 F F F 3 060-079 51 51 51 96 96 96 F F F 4 080-099 43 43 43 79 79 79 T T T 5 100-119 50 50 50 94 94 94 F F F 6 120-139 48 48 48 87 87 87 T T T 7 140-159 51 51 51 96 96 96 F F F 8 160-179 48 48 48 92 92 92 T T T 9 180-199 50 50 50 95 95 95 T T T 10 200-219 51 51 51 96 96 96 T T T 11 220-239 51 51 51 96 96 96 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 44 44 44 82 82 82 T T T 15 300-319 49 49 49 93 93 93 F F F 16 320-339 49 49 49 93 93 93 T T T 17 340-359 47 47 47 86 86 86 T T T 18 360-379 46 46 46 85 85 85 T T T 19 380-399 42 42 42 76 76 76 T T T 20 400-419 47 47 47 86 86 86 F F F 21 420-439 49 49 49 93 93 93 T T T 22 440-459 48 48 48 91 91 91 T T T 23 460-479 48 48 48 91 91 91 F F F 24 480-499 51 51 51 96 96 96 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 48 48 48 90 90 90 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 51, Max flush: 96 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 50 50 50 95 95 95 F F F 3 060-079 51 51 51 96 96 96 F F F 4 080-099 52 52 52 97 97 97 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 52 52 52 97 97 97 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 52 52 52 97 97 97 F F F 9 180-199 52 52 52 97 97 97 F F F 10 200-219 52 52 52 97 97 97 F F F 11 220-239 51 51 51 96 96 96 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 48 48 48 88 88 88 F F F 14 280-299 52 52 52 97 97 97 F F F 15 300-319 49 49 49 93 93 93 F F F 16 320-339 52 52 52 97 97 97 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 52 52 52 97 97 97 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 51 51 51 96 96 96 F F F 25 500-519 47 47 47 86 86 86 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 46 46 46 85 85 85 F F F 28 560-579 48 48 48 90 90 90 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 52, Max flush: 97 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.3556627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:16.400226267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.400607554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.433282837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.433853639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.433889104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.433902702Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.433921189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} {"msg":"Crucible 964e9fed-ff0f-4242-aeef-5b7e7fad385c has session id: a25674ef-9729-4063-8755-b9e31d8e6da4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.434585389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Upstairs opts: Upstairs UUID: 964e9fed-ff0f-4242-aeef-5b7e7fad385c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.434627453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Crucible stats registered with UUID: 964e9fed-ff0f-4242-aeef-5b7e7fad385c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.43465128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.465751607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.465783793Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} {"msg":"964e9fed-ff0f-4242-aeef-5b7e7fad385c active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.465852283Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.46587414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.465911465Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.465945171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.465978926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466012872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466044127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466077443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466110708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466149993Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_connection connected from Ok(127.0.0.1:43849)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466186588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_connection connected from Ok(127.0.0.1:59423)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466219893Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_connection connected from Ok(127.0.0.1:46437)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.4663136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466795534Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466832129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.466955972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.46697756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.46719147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.467232615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.467498918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.467533984Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.467814635Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468047453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468076149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468095627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468126882Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.46814173Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468155039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468309967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468331255Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[0]R flush_numbers[0..12]: [95, 84, 95, 96, 97, 94, 97, 96, 97, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468345222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[0]R generation[0..12]: [50, 46, 50, 51, 52, 50, 52, 51, 52, 52, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468359001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468372919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[1]R flush_numbers[0..12]: [95, 84, 95, 96, 97, 94, 97, 96, 97, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468386427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[1]R generation[0..12]: [50, 46, 50, 51, 52, 50, 52, 51, 52, 52, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468399775Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468413483Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[2]R flush_numbers[0..12]: [95, 84, 95, 96, 97, 94, 97, 96, 97, 97, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468430581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[2]R generation[0..12]: [50, 46, 50, 51, 52, 50, 52, 51, 52, 52, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468454518Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468482124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Max found gen is 53","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468504821Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Generation requested: 54 >= found:53","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:07:16.468529297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468558943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.46858409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"":"downstairs","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468600078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468611986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"0","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468625964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"1","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468648601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"client":"2","":"downstairs","session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468673418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468697254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"964e9fed-ff0f-4242-aeef-5b7e7fad385c is now active with session: a25674ef-9729-4063-8755-b9e31d8e6da4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468712512Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.46872714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398,"session_id":"a25674ef-9729-4063-8755-b9e31d8e6da4"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:16.468749017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 269 len 32768 data: 7 9 11 10 7 8 6 7 03/30 Flush 04/30 Write block 74 len 16384 data: 6 5 6 5 05/30 Write block 544 len 24576 data: 7 6 7 7 5 4 06/30 Write block 556 len 40960 data: 4 4 4 4 5 5 5 5 4 4 07/30 Read block 520 len 20480 08/30 Read block 414 len 24576 09/30 Write block 49 len 12288 data: 8 6 6 10/30 Read block 355 len 36864 11/30 Flush 12/30 Write block 298 len 28672 data: 6 7 5 4 4 3 3 13/30 Read block 281 len 8192 14/30 Read block 357 len 40960 15/30 Write block 316 len 8192 data: 5 6 16/30 Flush 17/30 Write block 3 len 32768 data: 4 5 4 4 4 4 4 5 18/30 Write block 387 len 8192 data: 9 10 19/30 Read block 527 len 32768 20/30 Write block 101 len 24576 data: 4 6 5 7 7 6 21/30 Read block 279 len 28672 22/30 Write block 511 len 28672 data: 2 2 2 3 3 4 4 23/30 Read block 445 len 36864 24/30 Read block 83 len 20480 25/30 Read block 565 len 12288 26/30 Read block 54 len 12288 27/30 Read block 522 len 16384 28/30 Write block 184 len 16384 data: 6 6 6 5 29/30 Read block 142 len 36864 30/30 Read block 250 len 32768 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 8 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:16.489382296Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 T T T 1 020-039 46 46 46 84 84 84 F F F 2 040-059 54 54 54 100 100 100 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 52 52 52 97 97 97 F F F 5 100-119 50 50 50 94 94 94 T T T 6 120-139 52 52 52 97 97 97 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 52 52 52 97 97 97 F F F 9 180-199 52 52 52 97 97 97 T T T 10 200-219 52 52 52 97 97 97 F F F 11 220-239 51 51 51 96 96 96 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 54 54 54 99 99 99 F F F 14 280-299 54 54 54 101 101 101 F F F 15 300-319 54 54 54 101 101 101 F F F 16 320-339 52 52 52 97 97 97 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 52 52 52 97 97 97 T T T 20 400-419 47 47 47 86 86 86 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 51 51 51 96 96 96 F F F 25 500-519 47 47 47 86 86 86 T T T 26 520-539 51 51 51 96 96 96 F F F 27 540-559 54 54 54 100 100 100 F F F 28 560-579 54 54 54 100 100 100 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 54, Max flush: 101 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 102 102 102 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 54 54 54 100 100 100 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 52 52 52 97 97 97 F F F 5 100-119 55 55 55 102 102 102 F F F 6 120-139 52 52 52 97 97 97 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 52 52 52 97 97 97 F F F 9 180-199 55 55 55 102 102 102 F F F 10 200-219 52 52 52 97 97 97 F F F 11 220-239 51 51 51 96 96 96 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 54 54 54 99 99 99 F F F 14 280-299 54 54 54 101 101 101 F F F 15 300-319 54 54 54 101 101 101 F F F 16 320-339 52 52 52 97 97 97 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 55 55 55 102 102 102 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 51 51 51 96 96 96 F F F 25 500-519 55 55 55 102 102 102 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 54 54 54 100 100 100 F F F 28 560-579 54 54 54 100 100 100 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 55, Max flush: 102 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.796379221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.842398609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843133309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843174943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843190741Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843204879Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} {"msg":"Crucible 2ddf0c9e-e1f5-4c59-8e53-99b633805dfb has session id: 11b40099-53b3-4300-8c43-6078c4f7afce","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843768322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Upstairs opts: Upstairs UUID: 2ddf0c9e-e1f5-4c59-8e53-99b633805dfb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843806477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Crucible stats registered with UUID: 2ddf0c9e-e1f5-4c59-8e53-99b633805dfb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.843826044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875319038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875360312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} {"msg":"2ddf0c9e-e1f5-4c59-8e53-99b633805dfb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875428443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875470327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875517341Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875551356Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875586482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875619987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875652912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875685078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875718673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875752709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_connection connected from Ok(127.0.0.1:52459)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875803002Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_connection connected from Ok(127.0.0.1:39912)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.87588824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_connection connected from Ok(127.0.0.1:49561)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.875931064Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.876413448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.876445874Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.876461972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.876485239Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.876626209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.876666024Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877043932Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877082697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877356869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877596796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877622263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877636431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877710741Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877730058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877748076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877837014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877856101Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[0]R flush_numbers[0..12]: [102, 84, 100, 100, 97, 102, 97, 96, 97, 102, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877870049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[0]R generation[0..12]: [55, 46, 54, 54, 52, 55, 52, 51, 52, 55, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877883527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877897415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[1]R flush_numbers[0..12]: [102, 84, 100, 100, 97, 102, 97, 96, 97, 102, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877910804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[1]R generation[0..12]: [55, 46, 54, 54, 52, 55, 52, 51, 52, 55, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877923942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.87793761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[2]R flush_numbers[0..12]: [102, 84, 100, 100, 97, 102, 97, 96, 97, 102, 97, 96]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877951618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[2]R generation[0..12]: [55, 46, 54, 54, 52, 55, 52, 51, 52, 55, 52, 51]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877965506Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877979084Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.877992462Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878005451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Next flush: 103","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878020479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:07:20.878034127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"":"downstairs","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878052344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878064203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"0","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878077301Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"1","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878090259Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"client":"2","":"downstairs","session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878103377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878115406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"2ddf0c9e-e1f5-4c59-8e53-99b633805dfb is now active with session: 11b40099-53b3-4300-8c43-6078c4f7afce","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878127784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:20.878139432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406,"session_id":"11b40099-53b3-4300-8c43-6078c4f7afce"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:07:20.878151361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 292 len 40960 data: 11 10 7 6 5 7 7 8 6 5 02/30 Read block 450 len 12288 03/30 Write block 386 len 24576 data: 8 10 11 8 8 8 04/30 Write block 203 len 32768 data: 4 5 5 5 4 5 4 4 05/30 Read block 454 len 8192 06/30 Read block 112 len 24576 07/30 Write block 495 len 36864 data: 6 4 6 6 6 6 5 4 4 08/30 Read block 479 len 20480 09/30 Read block 60 len 20480 10/30 Read block 342 len 32768 11/30 Read block 341 len 4096 12/30 Read block 285 len 28672 13/30 Read block 494 len 40960 14/30 Read block 488 len 16384 15/30 Write block 316 len 36864 data: 6 7 7 8 7 7 7 6 6 16/30 Read block 336 len 24576 17/30 Flush 18/30 Write block 219 len 28672 data: 5 4 4 5 6 7 6 19/30 Write block 497 len 12288 data: 7 7 7 20/30 Write block 83 len 32768 data: 2 3 3 4 2 3 5 7 21/30 Write block 187 len 40960 data: 6 6 4 4 3 4 4 4 4 3 22/30 Write block 284 len 16384 data: 4 5 6 7 23/30 Write block 582 len 4096 data: 4 24/30 Read block 75 len 28672 25/30 Read block 303 len 12288 26/30 Write block 388 len 4096 data: 12 27/30 Write block 178 len 4096 data: 4 28/30 Read block 358 len 28672 29/30 Write block 493 len 36864 data: 8 7 7 5 8 8 8 7 6 30/30 Write block 543 len 36864 data: 6 8 7 8 8 6 5 4 5 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 7 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:20.898140367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 102 102 102 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 54 54 54 100 100 100 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 52 52 52 97 97 97 T T T 5 100-119 55 55 55 102 102 102 F F F 6 120-139 52 52 52 97 97 97 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 52 52 52 97 97 97 T T T 9 180-199 55 55 55 102 102 102 T T T 10 200-219 56 56 56 103 103 103 T T T 11 220-239 51 51 51 96 96 96 T T T 12 240-259 50 50 50 95 95 95 F F F 13 260-279 54 54 54 99 99 99 F F F 14 280-299 56 56 56 103 103 103 T T T 15 300-319 56 56 56 103 103 103 F F F 16 320-339 56 56 56 103 103 103 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 56 56 56 103 103 103 T T T 20 400-419 47 47 47 86 86 86 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 56 56 56 103 103 103 T T T 25 500-519 56 56 56 103 103 103 T T T 26 520-539 51 51 51 96 96 96 F F F 27 540-559 54 54 54 100 100 100 T T T 28 560-579 54 54 54 100 100 100 F F F 29 580-599 44 44 44 82 82 82 T T T Max gen: 56, Max flush: 103 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 102 102 102 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 54 54 54 100 100 100 F F F 3 060-079 54 54 54 100 100 100 F F F 4 080-099 57 57 57 104 104 104 F F F 5 100-119 55 55 55 102 102 102 F F F 6 120-139 52 52 52 97 97 97 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 57 57 57 104 104 104 F F F 10 200-219 57 57 57 104 104 104 F F F 11 220-239 57 57 57 104 104 104 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 54 54 54 99 99 99 F F F 14 280-299 57 57 57 104 104 104 F F F 15 300-319 56 56 56 103 103 103 F F F 16 320-339 56 56 56 103 103 103 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 57 57 57 104 104 104 F F F 20 400-419 47 47 47 86 86 86 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 57 57 57 104 104 104 F F F 25 500-519 57 57 57 104 104 104 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 57 57 57 104 104 104 F F F 28 560-579 54 54 54 100 100 100 F F F 29 580-599 57 57 57 104 104 104 F F F Max gen: 57, Max flush: 104 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.204902686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:25.248202316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.248600921Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.280533455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281108887Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281151351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281167859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281185356Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} {"msg":"Crucible ed0fe8ed-c706-4179-b666-e7fe18923894 has session id: 87c74d79-a30f-42cf-8f4f-21214b9d786c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281821479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Upstairs opts: Upstairs UUID: ed0fe8ed-c706-4179-b666-e7fe18923894, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281869123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Crucible stats registered with UUID: ed0fe8ed-c706-4179-b666-e7fe18923894","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.281893469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313230064Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.3132645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} {"msg":"ed0fe8ed-c706-4179-b666-e7fe18923894 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.31333471Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313364066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313391313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.31340866Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313425408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313441376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313457633Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313473351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313490059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313509846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61098)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313540082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58860)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313596964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_connection connected from Ok(127.0.0.1:47528)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.313771401Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.314223119Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.314256624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.314274482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.31429101Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.314563912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.314594218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315043976Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315073143Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.31508865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315629696Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315658732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.31567398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315759319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315781516Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315798013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315810442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.31582405Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[0]R flush_numbers[0..12]: [102, 84, 100, 100, 104, 102, 97, 96, 104, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315837858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[0]R generation[0..12]: [55, 46, 54, 54, 57, 55, 52, 51, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315851566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315865354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[1]R flush_numbers[0..12]: [102, 84, 100, 100, 104, 102, 97, 96, 104, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315878852Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[1]R generation[0..12]: [55, 46, 54, 54, 57, 55, 52, 51, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315892111Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315905759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[2]R flush_numbers[0..12]: [102, 84, 100, 100, 104, 102, 97, 96, 104, 104, 104, 104]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315919237Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[2]R generation[0..12]: [55, 46, 54, 54, 57, 55, 52, 51, 57, 57, 57, 57]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315932395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.315953632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:07:25.315978059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316002825Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Next flush: 105","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316022673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316035901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"":"downstairs","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316049009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316060728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"0","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316073636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"1","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316086604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"client":"2","":"downstairs","session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316099612Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316111601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"ed0fe8ed-c706-4179-b666-e7fe18923894 is now active with session: 87c74d79-a30f-42cf-8f4f-21214b9d786c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316127868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316140227Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414,"session_id":"87c74d79-a30f-42cf-8f4f-21214b9d786c"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:25.316152275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 209 len 24576 data: 5 5 4 3 4 3 02/30 Read block 98 len 4096 03/30 Read block 418 len 40960 04/30 Flush 05/30 Write block 329 len 8192 data: 6 4 06/30 Write block 57 len 12288 data: 5 5 7 07/30 Read block 228 len 4096 08/30 Read block 483 len 28672 09/30 Read block 277 len 32768 10/30 Write block 78 len 32768 data: 5 5 4 4 2 3 4 4 11/30 Write block 412 len 28672 data: 5 4 4 5 5 5 6 12/30 Write block 220 len 40960 data: 5 5 6 7 8 7 6 3 3 3 13/30 Write block 260 len 36864 data: 4 3 4 4 7 7 7 5 6 14/30 Read block 82 len 16384 15/30 Read block 400 len 28672 16/30 Read block 65 len 4096 17/30 Write block 260 len 8192 data: 5 4 18/30 Write block 86 len 24576 data: 5 3 4 6 8 6 19/30 Write block 128 len 36864 data: 7 9 8 8 7 5 4 4 4 20/30 Write block 57 len 40960 data: 6 6 8 8 7 4 4 4 6 7 21/30 Read block 485 len 24576 22/30 Write block 317 len 16384 data: 8 8 9 8 23/30 Read block 524 len 4096 24/30 Write block 53 len 36864 data: 5 5 6 6 7 7 9 9 8 25/30 Read block 553 len 32768 26/30 Read block 50 len 40960 27/30 Read block 46 len 12288 28/30 Write block 222 len 16384 data: 7 8 9 8 29/30 Read block 132 len 28672 30/30 Read block 518 len 36864 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Write 3 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 8 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 26 25 25 76 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:25.335986843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 102 102 102 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 54 54 54 100 100 100 T T T 3 060-079 54 54 54 100 100 100 T T T 4 080-099 57 57 57 104 104 104 T T T 5 100-119 55 55 55 102 102 102 F F F 6 120-139 52 52 52 97 97 97 T T T 7 140-159 51 51 51 96 96 96 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 57 57 57 104 104 104 F F F 10 200-219 58 58 58 105 105 105 F F F 11 220-239 57 57 57 104 104 104 T T T 12 240-259 50 50 50 95 95 95 F F F 13 260-279 54 54 54 99 99 99 T T T 14 280-299 57 57 57 104 104 104 F F F 15 300-319 56 56 56 103 103 103 T T T 16 320-339 56 56 56 103 103 103 T T T 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 57 57 57 104 104 104 F F F 20 400-419 47 47 47 86 86 86 T T T 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 57 57 57 104 104 104 F F F 25 500-519 57 57 57 104 104 104 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 57 57 57 104 104 104 F F F 28 560-579 54 54 54 100 100 100 F F F 29 580-599 57 57 57 104 104 104 F F F Max gen: 58, Max flush: 105 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 55 55 55 102 102 102 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 59 59 59 106 106 106 F F F 3 060-079 59 59 59 106 106 106 F F F 4 080-099 59 59 59 106 106 106 F F F 5 100-119 55 55 55 102 102 102 F F F 6 120-139 59 59 59 106 106 106 F F F 7 140-159 51 51 51 96 96 96 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 57 57 57 104 104 104 F F F 10 200-219 58 58 58 105 105 105 F F F 11 220-239 59 59 59 106 106 106 F F F 12 240-259 50 50 50 95 95 95 F F F 13 260-279 59 59 59 106 106 106 F F F 14 280-299 57 57 57 104 104 104 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 59 59 59 106 106 106 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 57 57 57 104 104 104 F F F 20 400-419 59 59 59 106 106 106 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 57 57 57 104 104 104 F F F 25 500-519 57 57 57 104 104 104 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 57 57 57 104 104 104 F F F 28 560-579 54 54 54 100 100 100 F F F 29 580-599 57 57 57 104 104 104 F F F Max gen: 59, Max flush: 106 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.668077048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.714034014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.714742497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.714783452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.7147973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.714813038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} {"msg":"Crucible 5176306e-db48-42d4-ba0b-76d200ac8c4e has session id: c0440514-0650-4661-8f71-db4602cb9256","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.715460609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Upstairs opts: Upstairs UUID: 5176306e-db48-42d4-ba0b-76d200ac8c4e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.715498524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Crucible stats registered with UUID: 5176306e-db48-42d4-ba0b-76d200ac8c4e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.715514612Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.746933836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.746963602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} {"msg":"5176306e-db48-42d4-ba0b-76d200ac8c4e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747042281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747065358Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747095254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747113841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747130159Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747146167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747162604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747178692Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.74719518Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747228415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_connection connected from Ok(127.0.0.1:54147)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747261211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_connection connected from Ok(127.0.0.1:48324)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747295826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_connection connected from Ok(127.0.0.1:33003)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.747329992Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748039904Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.74806858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748187384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.74822231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748249426Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748276072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748544615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748872031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.748909046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749149313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749175249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749190027Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749396189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749421955Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749447652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749518712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749538219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[0]R flush_numbers[0..12]: [102, 84, 106, 106, 106, 102, 106, 96, 104, 104, 105, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749552148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[0]R generation[0..12]: [55, 46, 59, 59, 59, 55, 59, 51, 57, 57, 58, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749565836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749579794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[1]R flush_numbers[0..12]: [102, 84, 106, 106, 106, 102, 106, 96, 104, 104, 105, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749593322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[1]R generation[0..12]: [55, 46, 59, 59, 59, 55, 59, 51, 57, 57, 58, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.74960643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749620278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[2]R flush_numbers[0..12]: [102, 84, 106, 106, 106, 102, 106, 96, 104, 104, 105, 106]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749633786Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[2]R generation[0..12]: [55, 46, 59, 59, 59, 55, 59, 51, 57, 57, 58, 59]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749647095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749660513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749673281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749689009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 107","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749707816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749720525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"":"downstairs","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749733533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749745351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"0","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749758449Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"1","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749771268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"client":"2","":"downstairs","session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749784096Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749795964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"5176306e-db48-42d4-ba0b-76d200ac8c4e is now active with session: c0440514-0650-4661-8f71-db4602cb9256","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749809173Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749820911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422,"session_id":"c0440514-0650-4661-8f71-db4602cb9256"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:29.749832859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 150 len 28672 data: 5 6 6 5 5 9 9 02/30 Read block 549 len 20480 03/30 Read block 563 len 32768 04/30 Write block 284 len 24576 data: 5 6 7 8 9 10 05/30 Write block 576 len 32768 data: 4 4 5 4 3 3 5 4 06/30 Write block 48 len 4096 data: 9 07/30 Read block 533 len 28672 08/30 Write block 585 len 20480 data: 5 4 4 5 4 09/30 Read block 575 len 24576 10/30 Write block 8 len 16384 data: 5 5 6 6 11/30 Read block 537 len 8192 12/30 Flush 13/30 Write block 243 len 16384 data: 7 7 6 6 14/30 Write block 218 len 12288 data: 4 6 6 15/30 Write block 449 len 40960 data: 7 6 6 7 7 4 3 3 3 4 16/30 Read block 430 len 40960 17/30 Read block 204 len 36864 18/30 Write block 216 len 8192 data: 3 3 19/30 Read block 301 len 40960 20/30 Read block 252 len 40960 21/30 Read block 512 len 4096 22/30 Read block 228 len 32768 23/30 Write block 78 len 40960 data: 6 6 5 5 3 4 5 5 6 4 24/30 Write block 187 len 20480 data: 7 7 5 5 4 25/30 Read block 59 len 36864 26/30 Write block 560 len 36864 data: 6 6 6 6 5 5 4 5 5 27/30 Read block 221 len 32768 28/30 Write block 481 len 32768 data: 4 5 4 3 3 4 3 3 29/30 Write block 569 len 16384 data: 5 3 2 2 30/30 Write block 257 len 8192 data: 4 5 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 4 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Read 10 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:29.769480303Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 60 60 60 107 107 107 F F F 3 060-079 59 59 59 106 106 106 T T T 4 080-099 59 59 59 106 106 106 T T T 5 100-119 55 55 55 102 102 102 F F F 6 120-139 59 59 59 106 106 106 F F F 7 140-159 60 60 60 107 107 107 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 57 57 57 104 104 104 T T T 10 200-219 58 58 58 105 105 105 T T T 11 220-239 59 59 59 106 106 106 T T T 12 240-259 50 50 50 95 95 95 T T T 13 260-279 59 59 59 106 106 106 F F F 14 280-299 60 60 60 107 107 107 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 59 59 59 106 106 106 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 57 57 57 104 104 104 F F F 20 400-419 59 59 59 106 106 106 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 52 52 52 97 97 97 T T T 23 460-479 48 48 48 91 91 91 F F F 24 480-499 57 57 57 104 104 104 T T T 25 500-519 57 57 57 104 104 104 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 57 57 57 104 104 104 F F F 28 560-579 60 60 60 107 107 107 T T T 29 580-599 60 60 60 107 107 107 F F F Max gen: 60, Max flush: 107 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 46 46 46 84 84 84 F F F 2 040-059 60 60 60 107 107 107 F F F 3 060-079 61 61 61 108 108 108 F F F 4 080-099 61 61 61 108 108 108 F F F 5 100-119 55 55 55 102 102 102 F F F 6 120-139 59 59 59 106 106 106 F F F 7 140-159 60 60 60 107 107 107 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 61 61 61 108 108 108 F F F 11 220-239 61 61 61 108 108 108 F F F 12 240-259 61 61 61 108 108 108 F F F 13 260-279 59 59 59 106 106 106 F F F 14 280-299 60 60 60 107 107 107 F F F 15 300-319 59 59 59 106 106 106 F F F 16 320-339 59 59 59 106 106 106 F F F 17 340-359 52 52 52 97 97 97 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 57 57 57 104 104 104 F F F 20 400-419 59 59 59 106 106 106 F F F 21 420-439 52 52 52 97 97 97 F F F 22 440-459 61 61 61 108 108 108 F F F 23 460-479 48 48 48 91 91 91 F F F 24 480-499 61 61 61 108 108 108 F F F 25 500-519 57 57 57 104 104 104 F F F 26 520-539 51 51 51 96 96 96 F F F 27 540-559 57 57 57 104 104 104 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 60 60 60 107 107 107 F F F Max gen: 61, Max flush: 108 Begin loop 31 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.066161909Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.11142192Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.112029427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.112071061Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.112086209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.112100557Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} {"msg":"Crucible 161b811f-1427-4750-98b5-aeff83e87234 has session id: f4b5c099-17ea-4042-b269-c03ea043aeb6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.112601709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Upstairs opts: Upstairs UUID: 161b811f-1427-4750-98b5-aeff83e87234, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.11266348Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Crucible stats registered with UUID: 161b811f-1427-4750-98b5-aeff83e87234","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.112690147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144209417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144240363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} {"msg":"161b811f-1427-4750-98b5-aeff83e87234 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144312503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14433359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144361346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144378244Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144394642Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14441063Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144434906Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144462352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144491739Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144521434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:54313)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144573687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:50444)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144610562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_connection connected from Ok(127.0.0.1:39570)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.144652917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.145451907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.145489202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14550706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.145525887Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.145573471Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.145599867Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146115806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146281674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14630544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146691718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146732432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146757789Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146833888Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146853545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146871853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14689707Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146911928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[0]R flush_numbers[0..12]: [107, 84, 107, 108, 108, 102, 106, 107, 104, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146929405Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[0]R generation[0..12]: [60, 46, 60, 61, 61, 55, 59, 60, 57, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146943623Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146957501Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[1]R flush_numbers[0..12]: [107, 84, 107, 108, 108, 102, 106, 107, 104, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14697092Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[1]R generation[0..12]: [60, 46, 60, 61, 61, 55, 59, 60, 57, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146984178Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.146999845Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[2]R flush_numbers[0..12]: [107, 84, 107, 108, 108, 102, 106, 107, 104, 108, 108, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147023722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[2]R generation[0..12]: [60, 46, 60, 61, 61, 55, 59, 60, 57, 61, 61, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14703974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147053638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147066267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147079885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Next flush: 109","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147094832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"All extents match","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-11T06:07:34.14711461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"":"downstairs","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147139146Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147162873Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"0","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.14718788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"1","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147213106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"client":"2","":"downstairs","session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147239073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147263289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"161b811f-1427-4750-98b5-aeff83e87234 is now active with session: f4b5c099-17ea-4042-b269-c03ea043aeb6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147289536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"Set Active after no reconciliation"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147316992Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430,"session_id":"f4b5c099-17ea-4042-b269-c03ea043aeb6"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:34.147342249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 28672 data: 3 4 6 7 8 9 10 02/30 Flush 03/30 Write block 436 len 16384 data: 7 3 3 4 04/30 Write block 22 len 16384 data: 5 4 4 4 05/30 Write block 528 len 8192 data: 6 5 06/30 Write block 550 len 4096 data: 5 07/30 Write block 317 len 32768 data: 9 9 10 9 8 8 7 7 08/30 Read block 344 len 12288 09/30 Write block 590 len 20480 data: 4 4 3 2 2 10/30 Flush 11/30 Write block 384 len 24576 data: 6 6 9 11 13 9 12/30 Write block 133 len 36864 data: 6 5 5 5 5 5 5 6 6 13/30 Read block 525 len 40960 14/30 Read block 305 len 28672 15/30 Write block 500 len 28672 data: 8 7 5 5 6 5 5 16/30 Read block 453 len 8192 17/30 Flush 18/30 Write block 212 len 24576 data: 4 5 4 4 4 4 19/30 Read block 431 len 40960 20/30 Read block 557 len 28672 21/30 Write block 457 len 28672 data: 4 5 3 3 3 3 3 22/30 Write block 339 len 40960 data: 10 10 10 7 7 7 8 5 6 6 23/30 Read block 169 len 20480 24/30 Read block 261 len 16384 25/30 Write block 263 len 24576 data: 5 8 8 8 6 7 26/30 Read block 246 len 28672 27/30 Write block 24 len 8192 data: 5 5 28/30 Write block 537 len 36864 data: 3 2 3 4 6 6 7 9 8 29/30 Read block 161 len 8192 30/30 Read block 314 len 36864 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 13 12 13 38 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:34.167760567Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 62 62 62 110 110 110 T T T 2 040-059 60 60 60 107 107 107 F F F 3 060-079 61 61 61 108 108 108 F F F 4 080-099 61 61 61 108 108 108 F F F 5 100-119 55 55 55 102 102 102 F F F 6 120-139 62 62 62 111 111 111 F F F 7 140-159 62 62 62 111 111 111 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 61 61 61 108 108 108 T T T 11 220-239 61 61 61 108 108 108 F F F 12 240-259 61 61 61 108 108 108 F F F 13 260-279 59 59 59 106 106 106 T T T 14 280-299 62 62 62 109 109 109 F F F 15 300-319 62 62 62 110 110 110 F F F 16 320-339 62 62 62 110 110 110 T T T 17 340-359 52 52 52 97 97 97 T T T 18 360-379 52 52 52 97 97 97 F F F 19 380-399 62 62 62 111 111 111 F F F 20 400-419 59 59 59 106 106 106 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 61 61 61 108 108 108 T T T 23 460-479 48 48 48 91 91 91 T T T 24 480-499 61 61 61 108 108 108 F F F 25 500-519 62 62 62 111 111 111 F F F 26 520-539 62 62 62 110 110 110 T T T 27 540-559 62 62 62 110 110 110 T T T 28 560-579 61 61 61 108 108 108 F F F 29 580-599 62 62 62 110 110 110 F F F Max gen: 62, Max flush: 111 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 63 63 63 112 112 112 F F F 2 040-059 60 60 60 107 107 107 F F F 3 060-079 61 61 61 108 108 108 F F F 4 080-099 61 61 61 108 108 108 F F F 5 100-119 55 55 55 102 102 102 F F F 6 120-139 62 62 62 111 111 111 F F F 7 140-159 62 62 62 111 111 111 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 63 63 63 112 112 112 F F F 11 220-239 61 61 61 108 108 108 F F F 12 240-259 61 61 61 108 108 108 F F F 13 260-279 63 63 63 112 112 112 F F F 14 280-299 62 62 62 109 109 109 F F F 15 300-319 62 62 62 110 110 110 F F F 16 320-339 63 63 63 112 112 112 F F F 17 340-359 63 63 63 112 112 112 F F F 18 360-379 52 52 52 97 97 97 F F F 19 380-399 62 62 62 111 111 111 F F F 20 400-419 59 59 59 106 106 106 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 63 63 63 112 112 112 F F F 24 480-499 61 61 61 108 108 108 F F F 25 500-519 62 62 62 111 111 111 F F F 26 520-539 63 63 63 112 112 112 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 62 62 62 110 110 110 F F F Max gen: 63, Max flush: 112 Begin loop 32 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.464847968Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.509612137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.51024998Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.510293384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.510307892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.51032198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} {"msg":"Crucible 2e3852f5-0459-45c4-ad76-d10008e28a34 has session id: 9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.510853147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Upstairs opts: Upstairs UUID: 2e3852f5-0459-45c4-ad76-d10008e28a34, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.510894752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Crucible stats registered with UUID: 2e3852f5-0459-45c4-ad76-d10008e28a34","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.510917389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542108464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542143129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} {"msg":"2e3852f5-0459-45c4-ad76-d10008e28a34 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542217639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542240276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542269462Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542298238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542329354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542360609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542492861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542535975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542606316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542643531Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42924)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542676926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:39824)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542710522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_connection connected from Ok(127.0.0.1:39157)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.542743627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.54330905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.543351814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.54338118Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.543407876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.543436772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.543462999Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.543984497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544027042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544136136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544626069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544659375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544674763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544686871Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544700679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544714948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544726866Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544742984Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[0]R flush_numbers[0..12]: [107, 112, 107, 108, 108, 102, 111, 111, 104, 108, 112, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544756812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[0]R generation[0..12]: [60, 63, 60, 61, 61, 55, 62, 62, 57, 61, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.54477016Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544783948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[1]R flush_numbers[0..12]: [107, 112, 107, 108, 108, 102, 111, 111, 104, 108, 112, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544797486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[1]R generation[0..12]: [60, 63, 60, 61, 61, 55, 62, 62, 57, 61, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544810644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544824313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[2]R flush_numbers[0..12]: [107, 112, 107, 108, 108, 102, 111, 111, 104, 108, 112, 108]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544838001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[2]R generation[0..12]: [60, 63, 60, 61, 61, 55, 62, 62, 57, 61, 63, 61]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544851129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544864617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544877075Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544889884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544902342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544921519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"":"downstairs","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544936107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544948036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"0","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544960974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"1","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544973462Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"client":"2","":"downstairs","session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.54498621Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.544998279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"2e3852f5-0459-45c4-ad76-d10008e28a34 is now active with session: 9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.545010797Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.545022455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438,"session_id":"9cb7ddaf-f30c-4e03-b9ee-c1d4e56e99c9"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:38.545034274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 40960 data: 4 6 9 8 9 10 8 8 10 9 02/30 Write block 145 len 16384 data: 8 6 6 5 03/30 Write block 393 len 32768 data: 8 7 7 5 6 6 5 3 04/30 Read block 44 len 24576 05/30 Write block 582 len 24576 data: 6 5 5 6 5 5 06/30 Write block 114 len 20480 data: 6 6 4 4 3 07/30 Flush 08/30 Write block 233 len 40960 data: 3 5 7 7 7 6 6 7 8 7 09/30 Write block 366 len 40960 data: 8 8 7 7 7 6 5 5 5 6 10/30 Read block 362 len 16384 11/30 Read block 312 len 40960 12/30 Read block 493 len 12288 13/30 Write block 513 len 12288 data: 3 4 4 14/30 Flush 15/30 Flush 16/30 Write block 581 len 16384 data: 4 7 6 6 17/30 Write block 521 len 32768 data: 6 7 8 7 7 6 7 7 18/30 Read block 317 len 8192 19/30 Read block 12 len 24576 20/30 Write block 238 len 8192 data: 7 7 21/30 Read block 30 len 12288 22/30 Write block 252 len 32768 data: 5 4 5 4 5 5 6 5 23/30 Write block 332 len 40960 data: 5 6 4 5 7 6 7 11 11 11 24/30 Write block 222 len 20480 data: 8 9 10 9 7 25/30 Flush 26/30 Write block 144 len 28672 data: 8 9 7 7 6 5 6 27/30 Read block 423 len 32768 28/30 Write block 494 len 16384 data: 8 8 6 9 29/30 Read block 543 len 4096 30/30 Read block 523 len 32768 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 8 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:38.564741389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 63 63 63 112 112 112 F F F 2 040-059 64 64 64 113 113 113 F F F 3 060-079 61 61 61 108 108 108 F F F 4 080-099 61 61 61 108 108 108 F F F 5 100-119 64 64 64 113 113 113 F F F 6 120-139 62 62 62 111 111 111 F F F 7 140-159 64 64 64 113 113 113 T T T 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 63 63 63 112 112 112 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 64 64 64 116 116 116 F F F 13 260-279 63 63 63 112 112 112 F F F 14 280-299 62 62 62 109 109 109 F F F 15 300-319 62 62 62 110 110 110 F F F 16 320-339 64 64 64 116 116 116 F F F 17 340-359 64 64 64 116 116 116 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 113 113 113 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 63 63 63 112 112 112 F F F 24 480-499 61 61 61 108 108 108 T T T 25 500-519 64 64 64 114 114 114 F F F 26 520-539 64 64 64 116 116 116 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 64 64 64 116 116 116 F F F Max gen: 64, Max flush: 116 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 63 63 63 112 112 112 F F F 2 040-059 64 64 64 113 113 113 F F F 3 060-079 61 61 61 108 108 108 F F F 4 080-099 61 61 61 108 108 108 F F F 5 100-119 64 64 64 113 113 113 F F F 6 120-139 62 62 62 111 111 111 F F F 7 140-159 65 65 65 117 117 117 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 63 63 63 112 112 112 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 64 64 64 116 116 116 F F F 13 260-279 63 63 63 112 112 112 F F F 14 280-299 62 62 62 109 109 109 F F F 15 300-319 62 62 62 110 110 110 F F F 16 320-339 64 64 64 116 116 116 F F F 17 340-359 64 64 64 116 116 116 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 113 113 113 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 63 63 63 112 112 112 F F F 24 480-499 65 65 65 117 117 117 F F F 25 500-519 64 64 64 114 114 114 F F F 26 520-539 64 64 64 116 116 116 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 64 64 64 116 116 116 F F F Max gen: 65, Max flush: 117 Begin loop 33 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.823748137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:42.867584473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.867964561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.900384898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.900947411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.900982536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.900997844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.901022441Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} {"msg":"Crucible 59d75ac5-aed3-4854-a86a-c0d5e472f532 has session id: 3a7d20e2-afc1-41b7-8faf-4e86fa258ad3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.901647276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Upstairs opts: Upstairs UUID: 59d75ac5-aed3-4854-a86a-c0d5e472f532, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.901694499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Crucible stats registered with UUID: 59d75ac5-aed3-4854-a86a-c0d5e472f532","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.901719036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933077728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933120262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} {"msg":"59d75ac5-aed3-4854-a86a-c0d5e472f532 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933191163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933223438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933253194Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933271061Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933288189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933304307Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933320535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933336333Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.93335303Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933388065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48900)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933423321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_connection connected from Ok(127.0.0.1:53760)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933456756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_connection connected from Ok(127.0.0.1:54861)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.933561872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934124955Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934154901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934173358Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934195695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934316359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934352224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.93474387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934775546Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.934790794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935346778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935370485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935383763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935395501Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935408499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935421227Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935433056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935445824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[0]R flush_numbers[0..12]: [107, 112, 113, 108, 108, 113, 111, 117, 104, 108, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935469451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[0]R generation[0..12]: [60, 63, 64, 61, 61, 64, 62, 65, 57, 61, 63, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935486978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935501137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[1]R flush_numbers[0..12]: [107, 112, 113, 108, 108, 113, 111, 117, 104, 108, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935514645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[1]R generation[0..12]: [60, 63, 64, 61, 61, 64, 62, 65, 57, 61, 63, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935528013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935541751Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[2]R flush_numbers[0..12]: [107, 112, 113, 108, 108, 113, 111, 117, 104, 108, 112, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935555139Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[2]R generation[0..12]: [60, 63, 64, 61, 61, 64, 62, 65, 57, 61, 63, 64]","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-11T06:07:42.935574737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935588465Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935601133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935614011Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Next flush: 118","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.93562659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935639318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"":"downstairs","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935652216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935663884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"0","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935678282Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"1","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935691111Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"client":"2","":"downstairs","session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935704109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935715947Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"59d75ac5-aed3-4854-a86a-c0d5e472f532 is now active with session: 3a7d20e2-afc1-41b7-8faf-4e86fa258ad3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935728416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935739964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446,"session_id":"3a7d20e2-afc1-41b7-8faf-4e86fa258ad3"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:42.935751782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 340 len 24576 data: 12 12 8 8 8 9 03/30 Read block 398 len 28672 04/30 Write block 207 len 36864 data: 5 6 6 6 5 5 6 5 5 05/30 Write block 127 len 8192 data: 7 8 06/30 Write block 331 len 40960 data: 4 6 7 5 6 8 7 8 12 13 07/30 Read block 153 len 40960 08/30 Write block 486 len 20480 data: 5 4 4 3 5 09/30 Read block 315 len 24576 10/30 Read block 33 len 4096 11/30 Flush 12/30 Write block 479 len 16384 data: 5 5 5 6 13/30 Write block 44 len 8192 data: 10 11 14/30 Read block 563 len 24576 15/30 Write block 272 len 8192 data: 11 8 16/30 Read block 475 len 20480 17/30 Write block 511 len 12288 data: 3 3 4 18/30 Read block 13 len 40960 19/30 Read block 518 len 4096 20/30 Read block 288 len 24576 21/30 Read block 555 len 8192 22/30 Write block 67 len 4096 data: 8 23/30 Read block 185 len 20480 24/30 Write block 100 len 12288 data: 4 5 7 25/30 Write block 97 len 24576 data: 5 5 5 5 6 8 26/30 Read block 590 len 32768 27/30 Read block 70 len 40960 28/30 Read block 374 len 4096 29/30 Read block 378 len 28672 30/30 Read block 21 len 32768 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 19 19 18 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:42.954572969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 63 63 63 112 112 112 F F F 2 040-059 64 64 64 113 113 113 T T T 3 060-079 61 61 61 108 108 108 T T T 4 080-099 61 61 61 108 108 108 T T T 5 100-119 64 64 64 113 113 113 T T T 6 120-139 66 66 66 119 119 119 F F F 7 140-159 65 65 65 117 117 117 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 66 66 66 119 119 119 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 64 64 64 116 116 116 F F F 13 260-279 63 63 63 112 112 112 T T T 14 280-299 62 62 62 109 109 109 F F F 15 300-319 62 62 62 110 110 110 F F F 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 113 113 113 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 63 63 63 112 112 112 T T T 24 480-499 66 66 66 119 119 119 T T T 25 500-519 64 64 64 114 114 114 T T T 26 520-539 64 64 64 116 116 116 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 64 64 64 116 116 116 F F F Max gen: 66, Max flush: 119 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 63 63 63 112 112 112 F F F 2 040-059 67 67 67 120 120 120 F F F 3 060-079 67 67 67 120 120 120 F F F 4 080-099 67 67 67 120 120 120 F F F 5 100-119 67 67 67 120 120 120 F F F 6 120-139 66 66 66 119 119 119 F F F 7 140-159 65 65 65 117 117 117 F F F 8 160-179 57 57 57 104 104 104 F F F 9 180-199 61 61 61 108 108 108 F F F 10 200-219 66 66 66 119 119 119 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 64 64 64 116 116 116 F F F 13 260-279 67 67 67 120 120 120 F F F 14 280-299 62 62 62 109 109 109 F F F 15 300-319 62 62 62 110 110 110 F F F 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 113 113 113 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 67 67 67 120 120 120 F F F 24 480-499 67 67 67 120 120 120 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 64 64 64 116 116 116 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 64 64 64 116 116 116 F F F Max gen: 67, Max flush: 120 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.279895361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.324711683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325434524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325476898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325492556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325506844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} {"msg":"Crucible c88b5732-296e-494f-9a99-bd0ba299fed6 has session id: 54987acf-6b29-4cca-b57b-4c8ed86f131d","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325867965Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Upstairs opts: Upstairs UUID: c88b5732-296e-494f-9a99-bd0ba299fed6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325909639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Crucible stats registered with UUID: c88b5732-296e-494f-9a99-bd0ba299fed6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.325932776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.356979281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357014476Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} {"msg":"c88b5732-296e-494f-9a99-bd0ba299fed6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357097855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357119862Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357143338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357158087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357171645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357184573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357197641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357210729Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357224187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357237376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_connection connected from Ok(127.0.0.1:51939)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.35757153Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43784)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357622723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_connection connected from Ok(127.0.0.1:50297)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.357640981Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358173018Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358201854Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358216922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.3582304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358248187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358272374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358790623Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358830608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.358981587Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.3594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359427306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359442804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359455762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359468851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359481889Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359494907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359507805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[0]R flush_numbers[0..12]: [107, 112, 120, 120, 120, 120, 119, 117, 104, 108, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359521543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[0]R generation[0..12]: [60, 63, 67, 67, 67, 67, 66, 65, 57, 61, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359535191Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359553489Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[1]R flush_numbers[0..12]: [107, 112, 120, 120, 120, 120, 119, 117, 104, 108, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359567457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[1]R generation[0..12]: [60, 63, 67, 67, 67, 67, 66, 65, 57, 61, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359580935Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359595373Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[2]R flush_numbers[0..12]: [107, 112, 120, 120, 120, 120, 119, 117, 104, 108, 119, 116]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:07:47.359609221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[2]R generation[0..12]: [60, 63, 67, 67, 67, 67, 66, 65, 57, 61, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359628809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359642587Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359655355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359668153Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359680951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.35969364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"":"downstairs","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359707538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359719556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"0","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359732564Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"1","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359747082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"client":"2","":"downstairs","session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.35976116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359773299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"c88b5732-296e-494f-9a99-bd0ba299fed6 is now active with session: 54987acf-6b29-4cca-b57b-4c8ed86f131d"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359789217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359801225Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454,"session_id":"54987acf-6b29-4cca-b57b-4c8ed86f131d"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:47.359813204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 178 len 40960 data: 5 4 4 5 7 8 7 7 7 8 02/30 Write block 273 len 24576 data: 9 9 7 8 7 7 03/30 Write block 62 len 12288 data: 5 5 5 04/30 Read block 153 len 12288 05/30 Write block 488 len 36864 data: 5 4 6 6 7 9 9 9 7 06/30 Write block 135 len 4096 data: 6 07/30 Flush 08/30 Write block 51 len 24576 data: 7 6 6 6 7 7 09/30 Read block 410 len 20480 10/30 Read block 215 len 4096 11/30 Read block 530 len 32768 12/30 Write block 296 len 20480 data: 6 8 8 9 7 13/30 Read block 572 len 24576 14/30 Read block 492 len 12288 15/30 Read block 39 len 36864 16/30 Read block 592 len 4096 17/30 Read block 424 len 40960 18/30 Write block 89 len 40960 data: 7 9 7 7 6 6 5 4 6 6 19/30 Write block 136 len 20480 data: 6 6 6 6 7 20/30 Write block 39 len 32768 data: 3 5 7 10 9 11 12 9 21/30 Read block 395 len 40960 22/30 Read block 312 len 20480 23/30 Write block 57 len 28672 data: 8 8 10 10 9 6 6 24/30 Read block 486 len 8192 25/30 Read block 280 len 24576 26/30 Read block 552 len 8192 27/30 Write block 580 len 16384 data: 4 5 8 7 28/30 Read block 69 len 16384 29/30 Write block 301 len 40960 data: 6 5 4 4 3 4 3 3 2 2 30/30 Read block 574 len 36864 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 6 Done Done Done false 1008 Acked Read 5 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 5 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 3 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 23 22 23 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:47.37995365Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 63 63 63 112 112 112 T T T 2 040-059 67 67 67 120 120 120 T T T 3 060-079 68 68 68 121 121 121 T T T 4 080-099 67 67 67 120 120 120 T T T 5 100-119 67 67 67 120 120 120 F F F 6 120-139 68 68 68 121 121 121 T T T 7 140-159 65 65 65 117 117 117 T T T 8 160-179 68 68 68 121 121 121 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 66 66 66 119 119 119 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 64 64 64 116 116 116 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 62 62 62 109 109 109 T T T 15 300-319 62 62 62 110 110 110 T T T 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 113 113 113 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 67 67 67 120 120 120 F F F 24 480-499 68 68 68 121 121 121 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 64 64 64 116 116 116 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 64 64 64 116 116 116 T T T Max gen: 68, Max flush: 121 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 69 69 69 122 122 122 F F F 2 040-059 69 69 69 122 122 122 F F F 3 060-079 69 69 69 122 122 122 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 67 67 67 120 120 120 F F F 6 120-139 69 69 69 122 122 122 F F F 7 140-159 69 69 69 122 122 122 F F F 8 160-179 68 68 68 121 121 121 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 66 66 66 119 119 119 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 64 64 64 116 116 116 F F F 13 260-279 68 68 68 121 121 121 F F F 14 280-299 69 69 69 122 122 122 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 64 64 64 114 114 114 F F F 19 380-399 64 64 64 113 113 113 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 63 63 63 112 112 112 F F F 23 460-479 67 67 67 120 120 120 F F F 24 480-499 68 68 68 121 121 121 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 64 64 64 116 116 116 F F F 27 540-559 63 63 63 112 112 112 F F F 28 560-579 61 61 61 108 108 108 F F F 29 580-599 69 69 69 122 122 122 F F F Max gen: 69, Max flush: 122 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.676754629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.723082995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.723752134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.723795777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.723811066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.723825204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} {"msg":"Crucible 2f913d09-71d7-4aea-8a12-4a16a25c197e has session id: 4e4e7a95-05dd-467d-937d-2c5b5a0a0faa","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.724316326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Upstairs opts: Upstairs UUID: 2f913d09-71d7-4aea-8a12-4a16a25c197e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.724446279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Crucible stats registered with UUID: 2f913d09-71d7-4aea-8a12-4a16a25c197e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.724490673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.755905497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.755937983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} {"msg":"2f913d09-71d7-4aea-8a12-4a16a25c197e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756008973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756035959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756065355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756082423Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756099051Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756115438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756140645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756170611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756201457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756234262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:62436)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756267118Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:57286)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756302403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_connection connected from Ok(127.0.0.1:54032)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.756337218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757032643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757059179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757077927Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757091475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757109553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757134869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757765083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757799038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.757826835Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.7583685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758397716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758412484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758424863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758438091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758451069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758463057Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758475876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[0]R flush_numbers[0..12]: [107, 122, 122, 122, 122, 120, 122, 122, 121, 121, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758489424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[0]R generation[0..12]: [60, 69, 69, 69, 69, 67, 69, 69, 68, 68, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758502932Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.75851995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[1]R flush_numbers[0..12]: [107, 122, 122, 122, 122, 120, 122, 122, 121, 121, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758540957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[1]R generation[0..12]: [60, 69, 69, 69, 69, 67, 69, 69, 68, 68, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758554875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758568443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[2]R flush_numbers[0..12]: [107, 122, 122, 122, 122, 120, 122, 122, 121, 121, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758581931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[2]R generation[0..12]: [60, 69, 69, 69, 69, 67, 69, 69, 68, 68, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758595369Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758609057Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758621846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758642263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758655141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.7586677Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"":"downstairs","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758680568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758692386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"0","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758705154Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"1","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758719732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"client":"2","":"downstairs","session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758732791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758744819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"2f913d09-71d7-4aea-8a12-4a16a25c197e is now active with session: 4e4e7a95-05dd-467d-937d-2c5b5a0a0faa","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758757407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758769126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462,"session_id":"4e4e7a95-05dd-467d-937d-2c5b5a0a0faa"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:51.758781104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 378 len 36864 data: 6 7 6 7 7 6 7 7 10 02/30 Write block 30 len 40960 data: 5 2 2 2 2 2 2 3 3 4 03/30 Read block 74 len 12288 04/30 Write block 362 len 40960 data: 7 7 7 7 9 9 8 8 8 7 05/30 Flush 06/30 Write block 593 len 8192 data: 3 3 07/30 Read block 433 len 24576 08/30 Read block 315 len 24576 09/30 Read block 415 len 16384 10/30 Write block 448 len 28672 data: 7 8 7 7 8 8 5 11/30 Write block 374 len 32768 data: 6 7 5 6 7 8 7 8 12/30 Write block 488 len 36864 data: 6 5 7 7 8 10 10 10 8 13/30 Read block 325 len 4096 14/30 Write block 256 len 40960 data: 6 6 7 6 6 5 5 6 9 9 15/30 Read block 19 len 28672 16/30 Read block 98 len 28672 17/30 Write block 553 len 36864 data: 5 6 4 5 5 5 5 7 7 18/30 Write block 474 len 8192 data: 6 6 19/30 Write block 527 len 36864 data: 8 8 6 5 4 5 4 5 5 20/30 Read block 178 len 16384 21/30 Read block 236 len 24576 22/30 Write block 173 len 8192 data: 6 5 23/30 Write block 462 len 4096 data: 4 24/30 Flush 25/30 Write block 591 len 12288 data: 5 4 4 26/30 Read block 578 len 16384 27/30 Read block 498 len 24576 28/30 Read block 289 len 40960 29/30 Write block 483 len 36864 data: 5 4 4 6 5 7 6 8 8 30/30 Write block 379 len 32768 data: 9 8 9 8 7 8 8 11 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:51.779032284Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 69 69 69 122 122 122 F F F 3 060-079 69 69 69 122 122 122 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 67 67 67 120 120 120 F F F 6 120-139 69 69 69 122 122 122 F F F 7 140-159 69 69 69 122 122 122 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 66 66 66 119 119 119 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 70 70 70 124 124 124 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 69 69 69 122 122 122 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 70 70 70 124 124 124 T T T 19 380-399 70 70 70 124 124 124 T T T 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 70 70 70 124 124 124 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 70 70 70 124 124 124 T T T 25 500-519 67 67 67 120 120 120 F F F 26 520-539 70 70 70 124 124 124 F F F 27 540-559 70 70 70 124 124 124 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 70 70 70 124 124 124 T T T Max gen: 70, Max flush: 124 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 70 70 70 123 123 123 F F F 2 040-059 69 69 69 122 122 122 F F F 3 060-079 69 69 69 122 122 122 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 67 67 67 120 120 120 F F F 6 120-139 69 69 69 122 122 122 F F F 7 140-159 69 69 69 122 122 122 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 66 66 66 119 119 119 F F F 11 220-239 64 64 64 116 116 116 F F F 12 240-259 70 70 70 124 124 124 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 69 69 69 122 122 122 F F F 15 300-319 69 69 69 122 122 122 F F F 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 71 71 71 125 125 125 F F F 20 400-419 64 64 64 113 113 113 F F F 21 420-439 62 62 62 110 110 110 F F F 22 440-459 70 70 70 124 124 124 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 71 71 71 125 125 125 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 70 70 70 124 124 124 F F F 27 540-559 70 70 70 124 124 124 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 71, Max flush: 125 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.054613216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.09919645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.09985684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.099898924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.099913692Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.09992788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} {"msg":"Crucible 1313b9fb-da82-4dec-9a17-c404131c6ad3 has session id: ff26862b-26a1-4f46-9782-d91611f1ff1e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.100495722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Upstairs opts: Upstairs UUID: 1313b9fb-da82-4dec-9a17-c404131c6ad3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.100536877Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Crucible stats registered with UUID: 1313b9fb-da82-4dec-9a17-c404131c6ad3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.100559813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132301604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132333689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} {"msg":"1313b9fb-da82-4dec-9a17-c404131c6ad3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132368904Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132385812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132413998Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132432415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132454412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.13247207Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132489378Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132507105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132525083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132541691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:35132)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132561768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47936)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132863377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39795)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.132916899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.133291918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.133331413Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.133364848Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.133396024Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.1335672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.133596676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134025738Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134058393Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134074591Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134550716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134578902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.13459325Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134659911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134678479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134691987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134704225Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134717373Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[0]R flush_numbers[0..12]: [107, 123, 122, 122, 122, 120, 122, 122, 124, 121, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134730771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[0]R generation[0..12]: [60, 70, 69, 69, 69, 67, 69, 69, 70, 68, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134744239Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134757978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[1]R flush_numbers[0..12]: [107, 123, 122, 122, 122, 120, 122, 122, 124, 121, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134771636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[1]R generation[0..12]: [60, 70, 69, 69, 69, 67, 69, 69, 70, 68, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134788613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134802711Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[2]R flush_numbers[0..12]: [107, 123, 122, 122, 122, 120, 122, 122, 124, 121, 119, 116]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.13481619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[2]R generation[0..12]: [60, 70, 69, 69, 69, 67, 69, 69, 70, 68, 66, 64]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134829748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134843536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134857164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134870132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134882891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134895619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"":"downstairs","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134909187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134928804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"0","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134941883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"1","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134954611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"client":"2","":"downstairs","session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134967319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134979417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"1313b9fb-da82-4dec-9a17-c404131c6ad3 is now active with session: ff26862b-26a1-4f46-9782-d91611f1ff1e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.134994535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.135015812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470,"session_id":"ff26862b-26a1-4f46-9782-d91611f1ff1e"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:07:56.13503311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 388 len 4096 data: 14 02/30 Write block 26 len 16384 data: 4 5 6 5 03/30 Write block 536 len 20480 data: 4 4 3 4 5 04/30 Write block 405 len 8192 data: 5 6 05/30 Read block 11 len 28672 06/30 Read block 514 len 20480 07/30 Read block 360 len 8192 08/30 Read block 149 len 12288 09/30 Read block 459 len 36864 10/30 Read block 5 len 4096 11/30 Write block 254 len 8192 data: 6 5 12/30 Read block 388 len 16384 13/30 Flush 14/30 Write block 127 len 36864 data: 8 9 10 9 9 8 7 6 7 15/30 Write block 108 len 16384 data: 6 5 5 6 16/30 Read block 276 len 28672 17/30 Read block 79 len 28672 18/30 Write block 216 len 28672 data: 5 5 5 7 7 6 9 19/30 Write block 111 len 40960 data: 7 6 6 7 7 5 5 4 4 6 20/30 Read block 578 len 24576 21/30 Read block 596 len 12288 22/30 Read block 506 len 16384 23/30 Read block 180 len 8192 24/30 Write block 550 len 20480 data: 6 6 5 6 7 25/30 Write block 527 len 16384 data: 9 9 7 6 26/30 Write block 316 len 4096 data: 7 27/30 Read block 375 len 12288 28/30 Write block 26 len 12288 data: 5 6 7 29/30 Read block 230 len 12288 30/30 Write block 421 len 36864 data: 7 7 10 6 7 7 7 8 7 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 9 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 5 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:07:56.152678697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 72 72 72 126 126 126 T T T 2 040-059 69 69 69 122 122 122 F F F 3 060-079 69 69 69 122 122 122 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 67 67 67 120 120 120 T T T 6 120-139 69 69 69 122 122 122 T T T 7 140-159 69 69 69 122 122 122 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 66 66 66 119 119 119 T T T 11 220-239 64 64 64 116 116 116 T T T 12 240-259 72 72 72 126 126 126 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 69 69 69 122 122 122 F F F 15 300-319 69 69 69 122 122 122 T T T 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 72 72 72 126 126 126 F F F 21 420-439 62 62 62 110 110 110 T T T 22 440-459 70 70 70 124 124 124 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 71 71 71 125 125 125 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 72 72 72 126 126 126 T T T 27 540-559 72 72 72 126 126 126 T T T 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 72, Max flush: 126 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 69 69 69 122 122 122 F F F 3 060-079 69 69 69 122 122 122 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 73 73 73 127 127 127 F F F 6 120-139 73 73 73 127 127 127 F F F 7 140-159 69 69 69 122 122 122 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 73 73 73 127 127 127 F F F 11 220-239 73 73 73 127 127 127 F F F 12 240-259 72 72 72 126 126 126 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 69 69 69 122 122 122 F F F 15 300-319 73 73 73 127 127 127 F F F 16 320-339 66 66 66 119 119 119 F F F 17 340-359 66 66 66 119 119 119 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 72 72 72 126 126 126 F F F 20 400-419 72 72 72 126 126 126 F F F 21 420-439 73 73 73 127 127 127 F F F 22 440-459 70 70 70 124 124 124 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 71 71 71 125 125 125 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 73 73 73 127 127 127 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 73, Max flush: 127 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.448892597Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.494968297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.495618719Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.495668392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.495685819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.495703657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} {"msg":"Crucible 15a8e903-0a55-499c-8c6f-858970f3d8a8 has session id: fd9f6991-bdd1-455a-9ba7-6923a640900b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.49619106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Upstairs opts: Upstairs UUID: 15a8e903-0a55-499c-8c6f-858970f3d8a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.496278708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Crucible stats registered with UUID: 15a8e903-0a55-499c-8c6f-858970f3d8a8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.496303085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529419467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.52946724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} {"msg":"15a8e903-0a55-499c-8c6f-858970f3d8a8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529535491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529568466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529600392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.5296177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529634217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529650345Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529671332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529698979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529723155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529754471Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42872)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529786707Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_connection connected from Ok(127.0.0.1:33811)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529822162Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44642)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.529856697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.53049233Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.530535964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.530555602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.53056915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.530671855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.530698062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531062352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.53122231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531318497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531698715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531731491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531758327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531780784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531801161Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531815939Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531828508Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531842586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[0]R flush_numbers[0..12]: [107, 127, 122, 122, 122, 127, 127, 122, 124, 121, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531859453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[0]R generation[0..12]: [60, 73, 69, 69, 69, 73, 73, 69, 70, 68, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531873791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531888349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[1]R flush_numbers[0..12]: [107, 127, 122, 122, 122, 127, 127, 122, 124, 121, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531902477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[1]R generation[0..12]: [60, 73, 69, 69, 69, 73, 73, 69, 70, 68, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531915725Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531929454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[2]R flush_numbers[0..12]: [107, 127, 122, 122, 122, 127, 127, 122, 124, 121, 127, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531947261Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[2]R generation[0..12]: [60, 73, 69, 69, 69, 73, 73, 69, 70, 68, 73, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531964809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.531990535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532015552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532041018Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532065635Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532080813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"":"downstairs","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532094721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532110139Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"0","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2025-01-11T06:08:00.532140635Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"1","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532166281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"client":"2","":"downstairs","session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532188178Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532201816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"15a8e903-0a55-499c-8c6f-858970f3d8a8 is now active with session: fd9f6991-bdd1-455a-9ba7-6923a640900b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532214515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532233202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478,"session_id":"fd9f6991-bdd1-455a-9ba7-6923a640900b"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:00.532260078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 338 len 12288 data: 9 13 14 02/30 Read block 137 len 24576 03/30 Read block 478 len 20480 04/30 Read block 282 len 32768 05/30 Write block 57 len 32768 data: 9 9 11 11 10 7 7 6 06/30 Write block 492 len 32768 data: 9 11 11 11 9 10 9 9 07/30 Read block 362 len 16384 08/30 Write block 130 len 32768 data: 10 10 9 8 7 8 7 7 09/30 Write block 213 len 20480 data: 7 6 6 6 6 10/30 Read block 491 len 32768 11/30 Read block 172 len 4096 12/30 Write block 249 len 20480 data: 6 6 6 6 5 13/30 Read block 177 len 4096 14/30 Read block 146 len 40960 15/30 Write block 314 len 16384 data: 3 4 8 10 16/30 Read block 315 len 40960 17/30 Read block 158 len 24576 18/30 Read block 37 len 32768 19/30 Write block 445 len 24576 data: 8 8 6 8 9 8 20/30 Write block 118 len 20480 data: 5 5 7 6 6 21/30 Write block 412 len 24576 data: 6 5 5 6 6 6 22/30 Write block 286 len 16384 data: 9 10 11 11 23/30 Read block 113 len 12288 24/30 Flush 25/30 Write block 455 len 4096 data: 4 26/30 Write block 395 len 36864 data: 8 6 7 7 6 4 4 4 4 27/30 Read block 42 len 16384 28/30 Read block 572 len 32768 29/30 Write block 124 len 36864 data: 3 4 6 9 10 11 11 11 10 30/30 Read block 116 len 4096 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 1 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:00.552594458Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 74 74 74 128 128 128 F F F 3 060-079 74 74 74 128 128 128 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 74 74 74 128 128 128 F F F 6 120-139 74 74 74 128 128 128 T T T 7 140-159 69 69 69 122 122 122 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 73 73 73 127 127 127 F F F 12 240-259 74 74 74 128 128 128 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 74 74 74 128 128 128 F F F 15 300-319 74 74 74 128 128 128 F F F 16 320-339 74 74 74 128 128 128 F F F 17 340-359 74 74 74 128 128 128 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 72 72 72 126 126 126 T T T 20 400-419 74 74 74 128 128 128 T T T 21 420-439 73 73 73 127 127 127 F F F 22 440-459 74 74 74 128 128 128 T T T 23 460-479 70 70 70 124 124 124 F F F 24 480-499 74 74 74 128 128 128 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 73 73 73 127 127 127 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 74, Max flush: 128 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 74 74 74 128 128 128 F F F 3 060-079 74 74 74 128 128 128 F F F 4 080-099 69 69 69 122 122 122 F F F 5 100-119 74 74 74 128 128 128 F F F 6 120-139 75 75 75 129 129 129 F F F 7 140-159 69 69 69 122 122 122 F F F 8 160-179 70 70 70 124 124 124 F F F 9 180-199 68 68 68 121 121 121 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 73 73 73 127 127 127 F F F 12 240-259 74 74 74 128 128 128 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 74 74 74 128 128 128 F F F 15 300-319 74 74 74 128 128 128 F F F 16 320-339 74 74 74 128 128 128 F F F 17 340-359 74 74 74 128 128 128 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 75 75 75 129 129 129 F F F 20 400-419 75 75 75 129 129 129 F F F 21 420-439 73 73 73 127 127 127 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 74 74 74 128 128 128 F F F 25 500-519 67 67 67 120 120 120 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 73 73 73 127 127 127 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 75, Max flush: 129 Begin loop 38 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.823437568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.869089216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.869814937Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.869857711Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.869872849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.869891336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} {"msg":"Crucible cc67a27c-322f-49cd-8ab7-be59684753af has session id: 1552deec-05e9-4ed1-a563-4eafb5387b58","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.870399747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Upstairs opts: Upstairs UUID: cc67a27c-322f-49cd-8ab7-be59684753af, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.870497553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Crucible stats registered with UUID: cc67a27c-322f-49cd-8ab7-be59684753af","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.870517351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901423735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901454861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} {"msg":"cc67a27c-322f-49cd-8ab7-be59684753af active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901487446Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901503534Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.90153268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901552557Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901569735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901594211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901625267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901655013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901688249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901727203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_connection connected from Ok(127.0.0.1:50578)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901761219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_connection connected from Ok(127.0.0.1:63962)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.901793854Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_connection connected from Ok(127.0.0.1:63903)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.90182486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.902399261Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.902427868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.902443415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.902456833Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.902598584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.902674044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903077609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903107464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903122642Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903633073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903661269Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903680326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903762135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903781502Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.90379485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903807109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903820197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[0]R flush_numbers[0..12]: [107, 127, 128, 128, 122, 128, 129, 122, 124, 121, 128, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903833795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[0]R generation[0..12]: [60, 73, 74, 74, 69, 74, 75, 69, 70, 68, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903847403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903861201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[1]R flush_numbers[0..12]: [107, 127, 128, 128, 122, 128, 129, 122, 124, 121, 128, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.90387474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[1]R generation[0..12]: [60, 73, 74, 74, 69, 74, 75, 69, 70, 68, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903888758Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903902926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[2]R flush_numbers[0..12]: [107, 127, 128, 128, 122, 128, 129, 122, 124, 121, 128, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903916564Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[2]R generation[0..12]: [60, 73, 74, 74, 69, 74, 75, 69, 70, 68, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903930102Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.90394377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903959648Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903972866Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903985615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.903998253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"":"downstairs","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2025-01-11T06:08:04.90402074Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904033128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"0","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904046316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"1","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904059094Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"client":"2","":"downstairs","session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904072253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904084321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"cc67a27c-322f-49cd-8ab7-be59684753af is now active with session: 1552deec-05e9-4ed1-a563-4eafb5387b58","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904096949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904108768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486,"session_id":"1552deec-05e9-4ed1-a563-4eafb5387b58"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:04.904120746Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 340 len 4096 data: 15 02/30 Read block 91 len 32768 03/30 Flush 04/30 Write block 154 len 32768 data: 6 10 10 8 6 4 5 5 05/30 Write block 64 len 4096 data: 7 06/30 Write block 91 len 8192 data: 8 8 07/30 Write block 487 len 12288 data: 6 8 7 08/30 Read block 448 len 32768 09/30 Write block 311 len 32768 data: 2 2 3 4 5 9 11 10 10/30 Write block 296 len 28672 data: 7 9 9 10 8 7 6 11/30 Write block 508 len 12288 data: 4 4 3 12/30 Read block 370 len 36864 13/30 Flush 14/30 Write block 122 len 28672 data: 7 4 4 5 7 10 11 15/30 Flush 16/30 Write block 91 len 36864 data: 9 9 7 7 6 5 7 7 6 17/30 Write block 192 len 4096 data: 5 18/30 Write block 41 len 40960 data: 8 11 10 12 13 10 9 11 10 7 19/30 Read block 141 len 12288 20/30 Read block 285 len 32768 21/30 Read block 34 len 12288 22/30 Read block 333 len 36864 23/30 Write block 318 len 32768 data: 11 11 10 9 9 8 8 7 24/30 Write block 423 len 32768 data: 11 7 8 8 8 9 8 10 25/30 Read block 62 len 4096 26/30 Read block 186 len 20480 27/30 Read block 334 len 36864 28/30 Read block 130 len 20480 29/30 Read block 583 len 32768 30/30 Write block 353 len 24576 data: 7 5 6 5 4 4 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:04.923461881Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 74 74 74 128 128 128 T T T 3 060-079 76 76 76 131 131 131 F F F 4 080-099 76 76 76 131 131 131 T T T 5 100-119 74 74 74 128 128 128 F F F 6 120-139 76 76 76 132 132 132 F F F 7 140-159 76 76 76 131 131 131 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 68 68 68 121 121 121 T T T 10 200-219 74 74 74 128 128 128 F F F 11 220-239 73 73 73 127 127 127 F F F 12 240-259 74 74 74 128 128 128 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 76 76 76 131 131 131 T T T 16 320-339 74 74 74 128 128 128 T T T 17 340-359 76 76 76 130 130 130 T T T 18 360-379 71 71 71 125 125 125 F F F 19 380-399 75 75 75 129 129 129 F F F 20 400-419 75 75 75 129 129 129 F F F 21 420-439 73 73 73 127 127 127 T T T 22 440-459 75 75 75 129 129 129 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 76 76 76 131 131 131 F F F 25 500-519 76 76 76 131 131 131 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 73 73 73 127 127 127 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 76, Max flush: 132 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 77 77 77 133 133 133 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 77 77 77 133 133 133 F F F 5 100-119 74 74 74 128 128 128 F F F 6 120-139 76 76 76 132 132 132 F F F 7 140-159 76 76 76 131 131 131 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 77 77 77 133 133 133 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 73 73 73 127 127 127 F F F 12 240-259 74 74 74 128 128 128 F F F 13 260-279 70 70 70 124 124 124 F F F 14 280-299 76 76 76 131 131 131 F F F 15 300-319 77 77 77 133 133 133 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 77 77 77 133 133 133 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 75 75 75 129 129 129 F F F 20 400-419 75 75 75 129 129 129 F F F 21 420-439 77 77 77 133 133 133 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 70 70 70 124 124 124 F F F 24 480-499 76 76 76 131 131 131 F F F 25 500-519 76 76 76 131 131 131 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 73 73 73 127 127 127 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 77, Max flush: 133 Begin loop 39 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.21304167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:09.25640748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.256766411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.289152863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.289701478Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.289737203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.289750691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.289768599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} {"msg":"Crucible 0af349f4-3199-45dd-aec6-b5548f89e678 has session id: 4ba4deed-f112-438a-96d1-2f3def22d659","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.290332581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Upstairs opts: Upstairs UUID: 0af349f4-3199-45dd-aec6-b5548f89e678, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.290372376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Crucible stats registered with UUID: 0af349f4-3199-45dd-aec6-b5548f89e678","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.290388424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.321779072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.321809458Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} {"msg":"0af349f4-3199-45dd-aec6-b5548f89e678 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.321870249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.321903285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32193513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.321953248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.321976285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322007431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322039966Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322070632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322103507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322140112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_connection connected from Ok(127.0.0.1:42856)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322174558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_connection connected from Ok(127.0.0.1:49852)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322315908Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_connection connected from Ok(127.0.0.1:40954)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322362492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322806661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322846166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322876342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.322903318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.323075074Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32310433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.323594943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.323623239Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.323638108Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324178523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32420552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324223637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324236406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324249544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324262552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32427431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324287319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[0]R flush_numbers[0..12]: [107, 127, 133, 131, 133, 128, 132, 131, 131, 133, 128, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324301037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[0]R generation[0..12]: [60, 73, 77, 76, 77, 74, 76, 76, 76, 77, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324314545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324328463Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[1]R flush_numbers[0..12]: [107, 127, 133, 131, 133, 128, 132, 131, 131, 133, 128, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324341891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[1]R generation[0..12]: [60, 73, 77, 76, 77, 74, 76, 76, 76, 77, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32435501Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324368708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[2]R flush_numbers[0..12]: [107, 127, 133, 131, 133, 128, 132, 131, 131, 133, 128, 127]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324382106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[2]R generation[0..12]: [60, 73, 77, 76, 77, 74, 76, 76, 76, 77, 74, 73]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324395314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324410672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32442345Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324436288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Next flush: 134","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324448847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324461405Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"":"downstairs","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324474573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Transition from WaitQuorum to Active","v":0,"nameWait for a query_work_queue command to finish before sending IO ":"crucible","level":30,"time":"2025-01-11T06:08:09.32449475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"0","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324507918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"1","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324520857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"client":"2","":"downstairs","session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324533655Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324545663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"0af349f4-3199-45dd-aec6-b5548f89e678 is now active with session: 4ba4deed-f112-438a-96d1-2f3def22d659","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324558072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.32456983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494,"session_id":"4ba4deed-f112-438a-96d1-2f3def22d659"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:09.324581708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 100 len 28672 data: 6 7 9 6 8 8 7 03/30 Read block 4 len 12288 04/30 Read block 551 len 24576 05/30 Write block 87 len 16384 data: 5 5 8 10 06/30 Write block 249 len 32768 data: 7 7 7 7 6 7 6 7 07/30 Write block 342 len 28672 data: 9 9 9 10 6 7 7 08/30 Write block 488 len 4096 data: 9 09/30 Read block 160 len 32768 10/30 Write block 237 len 40960 data: 8 8 8 8 9 8 8 8 7 7 11/30 Read block 37 len 40960 12/30 Read block 469 len 24576 13/30 Flush 14/30 Write block 308 len 28672 data: 4 3 3 3 3 4 5 15/30 Write block 548 len 8192 data: 7 6 16/30 Write block 462 len 20480 data: 5 4 2 2 3 17/30 Write block 343 len 28672 data: 10 10 11 7 8 8 6 18/30 Read block 23 len 40960 19/30 Write block 304 len 32768 data: 5 4 5 4 5 4 4 4 20/30 Read block 536 len 36864 21/30 Read block 272 len 12288 22/30 Read block 14 len 4096 23/30 Read block 317 len 36864 24/30 Write block 292 len 28672 data: 12 11 8 7 8 10 10 25/30 Read block 74 len 32768 26/30 Write block 4 len 36864 data: 6 5 5 5 6 6 7 7 7 27/30 Write block 276 len 32768 data: 9 8 8 4 3 3 4 5 28/30 Write block 4 len 12288 data: 7 6 6 29/30 Read block 75 len 24576 30/30 Write block 340 len 16384 data: 16 13 10 11 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:09.344159132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 107 107 107 T T T 1 020-039 73 73 73 127 127 127 F F F 2 040-059 77 77 77 133 133 133 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 78 78 78 135 135 135 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 76 76 76 132 132 132 F F F 7 140-159 76 76 76 131 131 131 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 77 77 77 133 133 133 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 78 78 78 135 135 135 F F F 12 240-259 78 78 78 135 135 135 F F F 13 260-279 70 70 70 124 124 124 T T T 14 280-299 76 76 76 131 131 131 T T T 15 300-319 77 77 77 133 133 133 T T T 16 320-339 77 77 77 133 133 133 F F F 17 340-359 78 78 78 135 135 135 T T T 18 360-379 71 71 71 125 125 125 F F F 19 380-399 75 75 75 129 129 129 F F F 20 400-419 75 75 75 129 129 129 F F F 21 420-439 77 77 77 133 133 133 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 70 70 70 124 124 124 T T T 24 480-499 78 78 78 135 135 135 F F F 25 500-519 76 76 76 131 131 131 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 73 73 73 127 127 127 T T T 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 78, Max flush: 135 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 79 79 79 136 136 136 F F F 1 020-039 73 73 73 127 127 127 F F F 2 040-059 77 77 77 133 133 133 F F F 3 060-079 76 76 76 131 131 131 F F F 4 080-099 78 78 78 135 135 135 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 76 76 76 132 132 132 F F F 7 140-159 76 76 76 131 131 131 F F F 8 160-179 76 76 76 131 131 131 F F F 9 180-199 77 77 77 133 133 133 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 78 78 78 135 135 135 F F F 12 240-259 78 78 78 135 135 135 F F F 13 260-279 79 79 79 136 136 136 F F F 14 280-299 79 79 79 136 136 136 F F F 15 300-319 79 79 79 136 136 136 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 79 79 79 136 136 136 F F F 18 360-379 71 71 71 125 125 125 F F F 19 380-399 75 75 75 129 129 129 F F F 20 400-419 75 75 75 129 129 129 F F F 21 420-439 77 77 77 133 133 133 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 79 79 79 136 136 136 F F F 24 480-499 78 78 78 135 135 135 F F F 25 500-519 76 76 76 131 131 131 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 79 79 79 136 136 136 F F F 28 560-579 70 70 70 124 124 124 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 79, Max flush: 136 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.667997065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.713283123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.713944213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.713984537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.713998665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.714012903Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} {"msg":"Crucible 4dad375a-bedf-41f9-b4ef-908b5b17ef98 has session id: 2497afd0-8c72-41a3-b062-de53f4dc6e11","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.714523853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Upstairs opts: Upstairs UUID: 4dad375a-bedf-41f9-b4ef-908b5b17ef98, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.714554599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Crucible stats registered with UUID: 4dad375a-bedf-41f9-b4ef-908b5b17ef98","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.714574087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.74592009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.745967824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} {"msg":"4dad375a-bedf-41f9-b4ef-908b5b17ef98 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746034735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746061131Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746086978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746102645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746181125Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746226519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746260764Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746497671Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746533107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746557463Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_connection connected from Ok(127.0.0.1:64678)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746586479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_connection connected from Ok(127.0.0.1:41720)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746613886Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_connection connected from Ok(127.0.0.1:52387)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746641212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.746976826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747011971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747035768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747086381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747157021Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.7472385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747563956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747718175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.747768628Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748140777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748180341Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748205538Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748274959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748294376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748307874Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748319862Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748332891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[0]R flush_numbers[0..12]: [136, 127, 133, 131, 135, 135, 132, 131, 131, 133, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748346399Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[0]R generation[0..12]: [79, 73, 77, 76, 78, 78, 76, 76, 76, 77, 74, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748360057Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748373665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[1]R flush_numbers[0..12]: [136, 127, 133, 131, 135, 135, 132, 131, 131, 133, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748387253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[1]R generation[0..12]: [79, 73, 77, 76, 78, 78, 76, 76, 76, 77, 74, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748400421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748413849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[2]R flush_numbers[0..12]: [136, 127, 133, 131, 135, 135, 132, 131, 131, 133, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748427148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[2]R generation[0..12]: [79, 73, 77, 76, 78, 78, 76, 76, 76, 77, 74, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748440376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748453884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748469182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.74848203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748501577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748514276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"":"downstairs","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748527364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748539082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"0","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748552001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"1","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748564949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"client":"2","":"downstairs","session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748577937Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748589876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"4dad375a-bedf-41f9-b4ef-908b5b17ef98 is now active with session: 2497afd0-8c72-41a3-b062-de53f4dc6e11","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748602164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748613802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502,"session_id":"2497afd0-8c72-41a3-b062-de53f4dc6e11"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:13.748625791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 32768 data: 8 6 6 3 3 3 3 3 02/30 Write block 407 len 32768 data: 6 6 6 5 5 7 6 6 03/30 Write block 313 len 8192 data: 5 6 04/30 Write block 246 len 12288 data: 8 7 7 05/30 Write block 159 len 16384 data: 5 6 6 4 06/30 Read block 418 len 32768 07/30 Write block 243 len 36864 data: 9 9 8 9 8 8 8 8 8 08/30 Write block 56 len 4096 data: 8 09/30 Read block 375 len 8192 10/30 Read block 207 len 40960 11/30 Flush 12/30 Write block 71 len 4096 data: 6 13/30 Read block 459 len 28672 14/30 Flush 15/30 Write block 38 len 8192 data: 4 5 16/30 Read block 341 len 28672 17/30 Flush 18/30 Write block 143 len 32768 data: 8 9 10 8 8 7 6 7 19/30 Write block 14 len 40960 data: 8 8 9 7 9 9 8 6 6 5 20/30 Write block 258 len 40960 data: 8 7 7 6 6 7 10 10 9 7 21/30 Write block 148 len 12288 data: 8 7 8 22/30 Flush 23/30 Write block 276 len 20480 data: 10 9 9 5 4 24/30 Read block 397 len 16384 25/30 Write block 376 len 24576 data: 6 7 8 10 9 10 26/30 Write block 430 len 16384 data: 11 8 7 7 27/30 Read block 51 len 16384 28/30 Write block 153 len 36864 data: 6 7 11 11 9 7 6 7 7 29/30 Read block 378 len 24576 30/30 Write block 554 len 36864 data: 8 5 6 6 6 6 8 8 7 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 5 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:13.767139799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 137 137 137 F F F 3 060-079 80 80 80 138 138 138 F F F 4 080-099 78 78 78 135 135 135 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 76 76 76 132 132 132 F F F 7 140-159 80 80 80 140 140 140 T T T 8 160-179 80 80 80 137 137 137 T T T 9 180-199 77 77 77 133 133 133 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 78 78 78 135 135 135 F F F 12 240-259 80 80 80 140 140 140 F F F 13 260-279 80 80 80 140 140 140 T T T 14 280-299 79 79 79 136 136 136 T T T 15 300-319 80 80 80 137 137 137 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 79 79 79 136 136 136 F F F 18 360-379 71 71 71 125 125 125 T T T 19 380-399 75 75 75 129 129 129 T T T 20 400-419 80 80 80 137 137 137 F F F 21 420-439 77 77 77 133 133 133 T T T 22 440-459 75 75 75 129 129 129 F F F 23 460-479 79 79 79 136 136 136 F F F 24 480-499 78 78 78 135 135 135 F F F 25 500-519 76 76 76 131 131 131 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 79 79 79 136 136 136 T T T 28 560-579 70 70 70 124 124 124 T T T 29 580-599 71 71 71 125 125 125 F F F Max gen: 80, Max flush: 140 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 137 137 137 F F F 3 060-079 80 80 80 138 138 138 F F F 4 080-099 78 78 78 135 135 135 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 76 76 76 132 132 132 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 81 81 81 141 141 141 F F F 9 180-199 77 77 77 133 133 133 F F F 10 200-219 74 74 74 128 128 128 F F F 11 220-239 78 78 78 135 135 135 F F F 12 240-259 80 80 80 140 140 140 F F F 13 260-279 81 81 81 141 141 141 F F F 14 280-299 81 81 81 141 141 141 F F F 15 300-319 80 80 80 137 137 137 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 79 79 79 136 136 136 F F F 18 360-379 81 81 81 141 141 141 F F F 19 380-399 81 81 81 141 141 141 F F F 20 400-419 80 80 80 137 137 137 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 79 79 79 136 136 136 F F F 24 480-499 78 78 78 135 135 135 F F F 25 500-519 76 76 76 131 131 131 F F F 26 520-539 73 73 73 127 127 127 F F F 27 540-559 81 81 81 141 141 141 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 81, Max flush: 141 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.068117697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.11454014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.115163025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.115211818Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.115230755Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.115248963Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} {"msg":"Crucible 1748d245-664c-4fb3-8616-4b289eace92a has session id: 509b6f64-9f58-4354-b4a5-06f97c7c3198","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.115983642Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Upstairs opts: Upstairs UUID: 1748d245-664c-4fb3-8616-4b289eace92a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.116036115Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Crucible stats registered with UUID: 1748d245-664c-4fb3-8616-4b289eace92a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.116057412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147732381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147758068Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} {"msg":"1748d245-664c-4fb3-8616-4b289eace92a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147833178Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147866213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147897849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147917306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147934614Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147951501Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.147968119Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148025321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148061066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148244341Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_connection connected from Ok(127.0.0.1:63750)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148288535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_connection connected from Ok(127.0.0.1:47805)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148319641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_connection connected from Ok(127.0.0.1:53947)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148348347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148774799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148812514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.14883788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.148861787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.149008627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.149048541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.149430799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.149473983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.149743996Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150058933Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150086499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150108227Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150129624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150144271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.15015724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150322377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150342694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 137, 138, 135, 135, 132, 141, 141, 133, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150356782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[0]R generation[0..12]: [80, 80, 80, 80, 78, 78, 76, 81, 81, 77, 74, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.15037138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150385469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 137, 138, 135, 135, 132, 141, 141, 133, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150399037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[1]R generation[0..12]: [80, 80, 80, 80, 78, 78, 76, 81, 81, 77, 74, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150412275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150426033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 137, 138, 135, 135, 132, 141, 141, 133, 128, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150439411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[2]R generation[0..12]: [80, 80, 80, 80, 78, 78, 76, 81, 81, 77, 74, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150452829Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150466578Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150479226Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150498423Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150511061Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150526169Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"":"downstairs","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150539188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150550906Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"0","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150563664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"1","":"downstairs","session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150576432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.15059279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150605328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"1748d245-664c-4fb3-8616-4b289eace92a is now active with session: 509b6f64-9f58-4354-b4a5-06f97c7c3198","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150617987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150629855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510,"session_id":"509b6f64-9f58-4354-b4a5-06f97c7c3198"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:18.150641683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 258 len 8192 data: 9 8 02/30 Write block 205 len 32768 data: 6 6 6 7 7 7 6 6 03/30 Write block 241 len 8192 data: 10 9 04/30 Write block 64 len 4096 data: 8 05/30 Write block 89 len 8192 data: 9 11 06/30 Write block 255 len 20480 data: 7 8 7 10 9 07/30 Read block 272 len 4096 08/30 Read block 351 len 36864 09/30 Read block 549 len 8192 10/30 Write block 197 len 4096 data: 6 11/30 Write block 277 len 28672 data: 10 10 6 5 4 5 6 12/30 Read block 56 len 40960 13/30 Read block 523 len 24576 14/30 Read block 452 len 16384 15/30 Read block 208 len 8192 16/30 Write block 495 len 16384 data: 12 10 11 10 17/30 Read block 445 len 12288 18/30 Write block 206 len 8192 data: 7 7 19/30 Flush 20/30 Write block 129 len 36864 data: 12 12 12 11 9 8 9 8 8 21/30 Write block 374 len 28672 data: 7 8 7 8 9 11 10 22/30 Read block 119 len 40960 23/30 Flush 24/30 Flush 25/30 Write block 513 len 32768 data: 5 5 5 5 5 5 5 5 26/30 Write block 343 len 32768 data: 12 11 12 8 9 9 7 5 27/30 Write block 358 len 8192 data: 5 5 28/30 Read block 520 len 32768 29/30 Read block 395 len 12288 30/30 Write block 128 len 36864 data: 12 13 13 13 12 10 9 10 9 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:18.168184595Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 137 137 137 F F F 3 060-079 82 82 82 142 142 142 F F F 4 080-099 82 82 82 142 142 142 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 82 82 82 143 143 143 T T T 7 140-159 81 81 81 141 141 141 F F F 8 160-179 81 81 81 141 141 141 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 82 82 82 142 142 142 F F F 11 220-239 78 78 78 135 135 135 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 82 82 82 142 142 142 F F F 14 280-299 82 82 82 142 142 142 F F F 15 300-319 80 80 80 137 137 137 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 79 79 79 136 136 136 T T T 18 360-379 82 82 82 143 143 143 F F F 19 380-399 82 82 82 143 143 143 F F F 20 400-419 80 80 80 137 137 137 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 79 79 79 136 136 136 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 76 76 76 131 131 131 T T T 26 520-539 73 73 73 127 127 127 T T T 27 540-559 81 81 81 141 141 141 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 82, Max flush: 143 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 80 80 80 140 140 140 F F F 2 040-059 80 80 80 137 137 137 F F F 3 060-079 82 82 82 142 142 142 F F F 4 080-099 82 82 82 142 142 142 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 83 83 83 144 144 144 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 81 81 81 141 141 141 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 82 82 82 142 142 142 F F F 11 220-239 78 78 78 135 135 135 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 82 82 82 142 142 142 F F F 14 280-299 82 82 82 142 142 142 F F F 15 300-319 80 80 80 137 137 137 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 83 83 83 144 144 144 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 82 82 82 143 143 143 F F F 20 400-419 80 80 80 137 137 137 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 75 75 75 129 129 129 F F F 23 460-479 79 79 79 136 136 136 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 83 83 83 144 144 144 F F F 26 520-539 83 83 83 144 144 144 F F F 27 540-559 81 81 81 141 141 141 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 71 71 71 125 125 125 F F F Max gen: 83, Max flush: 144 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.440729591Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.485736418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.486352583Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.486393558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.486409336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.486424354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} {"msg":"Crucible 8778f255-ad9d-4a05-afaf-d5b47d4db065 has session id: 525c087f-702c-48fa-8fee-ea4e2fbd995d","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.48711029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Upstairs opts: Upstairs UUID: 8778f255-ad9d-4a05-afaf-d5b47d4db065, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.487157803Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Crucible stats registered with UUID: 8778f255-ad9d-4a05-afaf-d5b47d4db065","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.487184859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.518913661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.518945467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} {"msg":"8778f255-ad9d-4a05-afaf-d5b47d4db065 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519032885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519061181Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519090517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519118603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519151919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519176475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519200872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519226278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519252905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519277422Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_connection connected from Ok(127.0.0.1:54318)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519301398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_connection connected from Ok(127.0.0.1:63568)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519327845Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44388)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.519352091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520030668Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520058015Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520264267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520296242Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520318129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520343386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520632616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.520914828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521022403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521149615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521177352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.52119158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521378124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521403191Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521416849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521526384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521544561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[0]R flush_numbers[0..12]: [140, 140, 137, 142, 142, 135, 144, 141, 141, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521569398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[0]R generation[0..12]: [80, 80, 80, 82, 82, 78, 83, 81, 81, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521584746Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521603004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[1]R flush_numbers[0..12]: [140, 140, 137, 142, 142, 135, 144, 141, 141, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521616972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[1]R generation[0..12]: [80, 80, 80, 82, 82, 78, 83, 81, 81, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.52163015Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521643748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[2]R flush_numbers[0..12]: [140, 140, 137, 142, 142, 135, 144, 141, 141, 142, 142, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521657066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[2]R generation[0..12]: [80, 80, 80, 82, 82, 78, 83, 81, 81, 82, 82, 78]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521671024Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521684602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521696921Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521709549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521722807Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521735266Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"":"downstairs","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521750453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521780469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"0","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521806136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"1","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521831812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"client":"2","":"downstairs","session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.52184734Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521860918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"8778f255-ad9d-4a05-afaf-d5b47d4db065 is now active with session: 525c087f-702c-48fa-8fee-ea4e2fbd995d","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521882185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521904702Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518,"session_id":"525c087f-702c-48fa-8fee-ea4e2fbd995d"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:22.521927849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 342 len 8192 data: 11 13 02/30 Read block 345 len 40960 03/30 Write block 505 len 36864 data: 6 6 4 5 5 4 4 4 6 04/30 Read block 358 len 16384 05/30 Read block 136 len 40960 06/30 Write block 34 len 16384 data: 4 4 3 4 07/30 Read block 24 len 40960 08/30 Write block 460 len 8192 data: 4 4 09/30 Write block 539 len 16384 data: 5 6 7 7 10/30 Flush 11/30 Write block 214 len 20480 data: 7 7 7 7 6 12/30 Write block 59 len 20480 data: 12 12 11 8 8 13/30 Flush 14/30 Write block 453 len 28672 data: 9 6 5 4 5 6 4 15/30 Write block 409 len 16384 data: 7 6 6 8 16/30 Write block 449 len 8192 data: 10 9 17/30 Read block 88 len 16384 18/30 Write block 298 len 28672 data: 11 11 9 8 7 5 6 19/30 Read block 369 len 12288 20/30 Read block 525 len 32768 21/30 Write block 208 len 36864 data: 8 8 8 7 7 8 8 8 8 22/30 Write block 520 len 36864 data: 6 7 8 9 8 8 7 10 10 23/30 Write block 234 len 20480 data: 6 8 8 9 9 24/30 Read block 584 len 28672 25/30 Write block 31 len 32768 data: 4 4 4 5 5 4 5 5 26/30 Read block 396 len 28672 27/30 Write block 262 len 36864 data: 7 8 11 11 10 8 8 8 10 28/30 Write block 580 len 12288 data: 5 6 9 29/30 Read block 590 len 28672 30/30 Read block 360 len 24576 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Write 4 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 17 16 16 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:22.541786274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 84 84 84 145 145 145 T T T 2 040-059 84 84 84 146 146 146 F F F 3 060-079 84 84 84 146 146 146 F F F 4 080-099 82 82 82 142 142 142 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 83 83 83 144 144 144 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 81 81 81 141 141 141 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 84 84 84 146 146 146 T T T 11 220-239 78 78 78 135 135 135 T T T 12 240-259 82 82 82 142 142 142 F F F 13 260-279 82 82 82 142 142 142 T T T 14 280-299 82 82 82 142 142 142 T T T 15 300-319 80 80 80 137 137 137 T T T 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 82 82 82 143 143 143 F F F 20 400-419 80 80 80 137 137 137 T T T 21 420-439 81 81 81 141 141 141 F F F 22 440-459 75 75 75 129 129 129 T T T 23 460-479 84 84 84 145 145 145 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 84 84 84 145 145 145 F F F 26 520-539 84 84 84 145 145 145 T T T 27 540-559 84 84 84 145 145 145 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 71 71 71 125 125 125 T T T Max gen: 84, Max flush: 146 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 85 85 85 147 147 147 F F F 2 040-059 84 84 84 146 146 146 F F F 3 060-079 84 84 84 146 146 146 F F F 4 080-099 82 82 82 142 142 142 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 83 83 83 144 144 144 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 81 81 81 141 141 141 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 85 85 85 147 147 147 F F F 11 220-239 85 85 85 147 147 147 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 85 85 85 147 147 147 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 82 82 82 143 143 143 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 84 84 84 145 145 145 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 84 84 84 145 145 145 F F F 26 520-539 85 85 85 147 147 147 F F F 27 540-559 84 84 84 145 145 145 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 85, Max flush: 147 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.843405245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:26.887018592Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.887415558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.919322225Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.91986789Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.919904385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.919917764Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.919931442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} {"msg":"Crucible b58a294a-7b33-4793-9f05-c04b4973a4ce has session id: 88df8846-37e4-49e2-9cb1-83549f79287b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.920477297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Upstairs opts: Upstairs UUID: b58a294a-7b33-4793-9f05-c04b4973a4ce, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.920516562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Crucible stats registered with UUID: b58a294a-7b33-4793-9f05-c04b4973a4ce","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.92053247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951571995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951606771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} {"msg":"b58a294a-7b33-4793-9f05-c04b4973a4ce active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.95168549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951707977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951735613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951752361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951768979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951785097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951811363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.95183289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951864915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951895761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_connection connected from Ok(127.0.0.1:63009)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951928987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58302)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951960462Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58080)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.951993558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.952788349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.952817255Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.952831973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.952849191Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.952862759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.952876697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.953473066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.95351275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.953543206Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954123097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954159412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954181309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954196197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954209705Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954222703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954234691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.95424781Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[0]R flush_numbers[0..12]: [140, 147, 146, 146, 142, 135, 144, 141, 141, 142, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954261608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[0]R generation[0..12]: [80, 85, 84, 84, 82, 78, 83, 81, 81, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954278165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954292154Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[1]R flush_numbers[0..12]: [140, 147, 146, 146, 142, 135, 144, 141, 141, 142, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954305602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[1]R generation[0..12]: [80, 85, 84, 84, 82, 78, 83, 81, 81, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.9543188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954332688Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[2]R flush_numbers[0..12]: [140, 147, 146, 146, 142, 135, 144, 141, 141, 142, 147, 147]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954346156Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[2]R generation[0..12]: [80, 85, 84, 84, 82, 78, 83, 81, 81, 82, 85, 85]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954359794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954373423Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954385881Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954398609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Next flush: 148","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954411087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:08:26.954423556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"":"downstairs","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954441343Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954452942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"0","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.95446879Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"1","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954481938Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"client":"2","":"downstairs","session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954494876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954506934Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"b58a294a-7b33-4793-9f05-c04b4973a4ce is now active with session: 88df8846-37e4-49e2-9cb1-83549f79287b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954519432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954531041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526,"session_id":"88df8846-37e4-49e2-9cb1-83549f79287b"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:26.954543049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 172 len 24576 data: 7 7 6 5 4 5 02/30 Write block 469 len 24576 data: 4 4 3 4 4 7 03/30 Write block 216 len 12288 data: 9 8 7 04/30 Read block 92 len 8192 05/30 Read block 369 len 28672 06/30 Read block 562 len 28672 07/30 Read block 56 len 24576 08/30 Write block 56 len 28672 data: 9 10 10 13 13 12 9 09/30 Read block 400 len 8192 10/30 Write block 529 len 28672 data: 8 7 5 6 5 6 6 11/30 Write block 287 len 40960 data: 11 12 12 12 13 13 12 9 8 9 12/30 Read block 72 len 8192 13/30 Read block 305 len 4096 14/30 Read block 64 len 16384 15/30 Flush 16/30 Write block 207 len 24576 data: 8 9 9 9 8 8 17/30 Write block 33 len 20480 data: 5 6 6 5 6 18/30 Write block 228 len 24576 data: 4 4 4 4 3 4 19/30 Read block 476 len 28672 20/30 Read block 491 len 24576 21/30 Read block 166 len 24576 22/30 Write block 462 len 40960 data: 6 5 3 3 4 4 4 5 5 4 23/30 Flush 24/30 Write block 78 len 20480 data: 7 7 6 6 4 25/30 Flush 26/30 Write block 125 len 24576 data: 6 8 11 13 14 14 27/30 Write block 522 len 20480 data: 9 10 9 9 8 28/30 Write block 43 len 4096 data: 11 29/30 Write block 380 len 32768 data: 11 11 9 8 9 9 12 12 30/30 Read block 258 len 20480 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:26.974167946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 86 86 86 149 149 149 F F F 2 040-059 86 86 86 148 148 148 T T T 3 060-079 86 86 86 150 150 150 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 83 83 83 144 144 144 T T T 7 140-159 81 81 81 141 141 141 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 86 86 86 148 148 148 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 82 82 82 143 143 143 T T T 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 86 86 86 149 149 149 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 84 84 84 145 145 145 F F F 26 520-539 86 86 86 148 148 148 T T T 27 540-559 84 84 84 145 145 145 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 86, Max flush: 150 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 86 86 86 149 149 149 F F F 2 040-059 87 87 87 151 151 151 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 86 86 86 148 148 148 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 86 86 86 148 148 148 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 87 87 87 151 151 151 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 86 86 86 149 149 149 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 84 84 84 145 145 145 F F F 26 520-539 87 87 87 151 151 151 F F F 27 540-559 84 84 84 145 145 145 F F F 28 560-579 81 81 81 141 141 141 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 87, Max flush: 151 Begin loop 44 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.278786837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.323645954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.32433128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.324377204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.324392072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.32440659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} {"msg":"Crucible 1bf4cb82-28b1-4bbc-956b-035ae4e62a80 has session id: b28b04b3-1810-47fe-9f36-a1763f061ecf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.32491726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Upstairs opts: Upstairs UUID: 1bf4cb82-28b1-4bbc-956b-035ae4e62a80, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.324958874Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Crucible stats registered with UUID: 1bf4cb82-28b1-4bbc-956b-035ae4e62a80","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.324982601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357221793Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357256828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} {"msg":"1bf4cb82-28b1-4bbc-956b-035ae4e62a80 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357334778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357352985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357375392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.35738938Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357407468Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357421126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357434254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357454161Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357472928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357490386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:50289)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357574525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:63565)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357604241Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43412)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.357631267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358328802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358366896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358395862Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358423989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358451905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358478221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.358957565Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359152519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359179095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359609056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359645431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359671688Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359732439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359761566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359777213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359790632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.35980406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[0]R flush_numbers[0..12]: [140, 149, 151, 150, 150, 135, 151, 141, 148, 142, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359817778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[0]R generation[0..12]: [80, 86, 87, 86, 86, 78, 87, 81, 86, 82, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359831516Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359845304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[1]R flush_numbers[0..12]: [140, 149, 151, 150, 150, 135, 151, 141, 148, 142, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359859022Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[1]R generation[0..12]: [80, 86, 87, 86, 86, 78, 87, 81, 86, 82, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.35987638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359890648Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[2]R flush_numbers[0..12]: [140, 149, 151, 150, 150, 135, 151, 141, 148, 142, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359904216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[2]R generation[0..12]: [80, 86, 87, 86, 86, 78, 87, 81, 86, 82, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359917454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359931023Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359943641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359956479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Next flush: 152","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359969217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359981876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"":"downstairs","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.359995084Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360006912Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"0","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360028709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"1","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360041757Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"client":"2","":"downstairs","session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360054686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360066674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"1bf4cb82-28b1-4bbc-956b-035ae4e62a80 is now active with session: b28b04b3-1810-47fe-9f36-a1763f061ecf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360081772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.36009375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534,"session_id":"b28b04b3-1810-47fe-9f36-a1763f061ecf"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:31.360105839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 515 len 12288 data: 6 6 6 02/30 Write block 573 len 4096 data: 2 03/30 Read block 468 len 40960 04/30 Read block 133 len 8192 05/30 Write block 476 len 12288 data: 5 6 6 06/30 Write block 530 len 8192 data: 8 6 07/30 Write block 460 len 20480 data: 5 5 7 6 4 08/30 Read block 117 len 36864 09/30 Read block 6 len 40960 10/30 Read block 394 len 16384 11/30 Read block 134 len 12288 12/30 Read block 432 len 20480 13/30 Read block 167 len 36864 14/30 Flush 15/30 Write block 56 len 12288 data: 10 11 11 16/30 Write block 561 len 40960 data: 9 8 7 6 6 5 6 6 6 4 17/30 Write block 168 len 32768 data: 6 6 7 6 8 8 7 6 18/30 Read block 28 len 16384 19/30 Read block 434 len 16384 20/30 Read block 568 len 24576 21/30 Read block 201 len 16384 22/30 Read block 122 len 20480 23/30 Read block 136 len 8192 24/30 Read block 159 len 28672 25/30 Flush 26/30 Write block 471 len 36864 data: 5 5 5 8 7 6 7 7 6 27/30 Write block 476 len 16384 data: 7 8 8 7 28/30 Read block 571 len 8192 29/30 Read block 57 len 36864 30/30 Read block 83 len 16384 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:31.379228304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 86 86 86 149 149 149 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 88 88 88 153 153 153 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 86 86 86 148 148 148 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 87 87 87 151 151 151 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 88 88 88 152 152 152 T T T 24 480-499 82 82 82 142 142 142 F F F 25 500-519 88 88 88 152 152 152 F F F 26 520-539 88 88 88 152 152 152 F F F 27 540-559 84 84 84 145 145 145 F F F 28 560-579 88 88 88 153 153 153 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 88, Max flush: 153 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 140 140 140 F F F 1 020-039 86 86 86 149 149 149 F F F 2 040-059 88 88 88 153 153 153 F F F 3 060-079 86 86 86 150 150 150 F F F 4 080-099 86 86 86 150 150 150 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 88 88 88 153 153 153 F F F 9 180-199 82 82 82 142 142 142 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 86 86 86 148 148 148 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 82 82 82 143 143 143 F F F 19 380-399 87 87 87 151 151 151 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 89 89 89 154 154 154 F F F 24 480-499 82 82 82 142 142 142 F F F 25 500-519 88 88 88 152 152 152 F F F 26 520-539 88 88 88 152 152 152 F F F 27 540-559 84 84 84 145 145 145 F F F 28 560-579 88 88 88 153 153 153 F F F 29 580-599 85 85 85 147 147 147 F F F Max gen: 89, Max flush: 154 Begin loop 45 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.635271828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.680183007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.680944433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.680997736Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.681019593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.68103869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} {"msg":"Crucible 9878c7bd-e5ae-43ad-b661-1ee2db88d6d0 has session id: e2f2a9b4-e2b0-4126-9b06-67de6fd289c5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.68169545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Upstairs opts: Upstairs UUID: 9878c7bd-e5ae-43ad-b661-1ee2db88d6d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.681743874Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Crucible stats registered with UUID: 9878c7bd-e5ae-43ad-b661-1ee2db88d6d0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.68176989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713449718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713481534Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} {"msg":"9878c7bd-e5ae-43ad-b661-1ee2db88d6d0 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713562133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.71358439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713612626Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713640292Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713671128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713700704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.71373299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713764475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.71380276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713837425Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:45536)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713867681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:52333)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713899567Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:56476)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.713931853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.714646155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.714684819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.714710816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.714733653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.714884832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.714915998Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.715438436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.715469772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.715495988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716081439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716112994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716128812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.7161414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716155058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716168876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716181615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716194503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[0]R flush_numbers[0..12]: [140, 149, 153, 150, 150, 135, 151, 141, 153, 142, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716207891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[0]R generation[0..12]: [80, 86, 88, 86, 86, 78, 87, 81, 88, 82, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716221919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716236377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[1]R flush_numbers[0..12]: [140, 149, 153, 150, 150, 135, 151, 141, 153, 142, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716250125Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[1]R generation[0..12]: [80, 86, 88, 86, 86, 78, 87, 81, 88, 82, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716263424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716277092Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[2]R flush_numbers[0..12]: [140, 149, 153, 150, 150, 135, 151, 141, 153, 142, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.71629074Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[2]R generation[0..12]: [80, 86, 88, 86, 86, 78, 87, 81, 88, 82, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716304048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716320396Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716333314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716346722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.71635943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716377878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"":"downstairs","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716391166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716410983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"0","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716426211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"1","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716439559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"client":"2","":"downstairs","session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716452668Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716464766Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"9878c7bd-e5ae-43ad-b661-1ee2db88d6d0 is now active with session: e2f2a9b4-e2b0-4126-9b06-67de6fd289c5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716477214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716488853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542,"session_id":"e2f2a9b4-e2b0-4126-9b06-67de6fd289c5"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:35.716500691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 54 len 40960 data: 7 8 11 12 12 14 14 13 10 9 02/30 Write block 375 len 16384 data: 9 8 9 10 03/30 Read block 487 len 40960 04/30 Read block 160 len 32768 05/30 Write block 13 len 28672 data: 7 9 9 10 8 10 10 06/30 Flush 07/30 Write block 0 len 32768 data: 2 3 4 5 8 7 7 6 08/30 Read block 265 len 28672 09/30 Write block 529 len 12288 data: 9 9 7 10/30 Flush 11/30 Write block 167 len 12288 data: 6 7 7 12/30 Flush 13/30 Flush 14/30 Write block 587 len 40960 data: 6 6 5 5 6 5 5 4 2 2 15/30 Write block 285 len 4096 data: 8 16/30 Read block 562 len 16384 17/30 Read block 560 len 4096 18/30 Flush 19/30 Write block 532 len 36864 data: 7 6 7 7 5 5 4 6 7 20/30 Write block 495 len 28672 data: 13 11 12 11 10 9 8 21/30 Read block 338 len 36864 22/30 Read block 397 len 16384 23/30 Read block 369 len 24576 24/30 Read block 244 len 20480 25/30 Read block 390 len 8192 26/30 Write block 195 len 20480 data: 5 4 7 6 5 27/30 Write block 46 len 4096 data: 11 28/30 Read block 222 len 36864 29/30 Write block 483 len 20480 data: 6 5 5 7 7 30/30 Write block 90 len 24576 data: 12 10 10 8 8 7 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:35.735710165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 F F F 1 020-039 86 86 86 149 149 149 F F F 2 040-059 90 90 90 155 155 155 T T T 3 060-079 90 90 90 155 155 155 F F F 4 080-099 86 86 86 150 150 150 T T T 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 82 82 82 142 142 142 T T T 10 200-219 86 86 86 149 149 149 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 90 90 90 159 159 159 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 90 90 90 155 155 155 F F F 19 380-399 87 87 87 151 151 151 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 89 89 89 154 154 154 F F F 24 480-499 82 82 82 142 142 142 T T T 25 500-519 88 88 88 152 152 152 T T T 26 520-539 90 90 90 156 156 156 T T T 27 540-559 84 84 84 145 145 145 T T T 28 560-579 88 88 88 153 153 153 F F F 29 580-599 90 90 90 159 159 159 F F F Max gen: 90, Max flush: 159 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 F F F 1 020-039 86 86 86 149 149 149 F F F 2 040-059 91 91 91 160 160 160 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 91 91 91 160 160 160 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 81 81 81 141 141 141 F F F 8 160-179 90 90 90 157 157 157 F F F 9 180-199 91 91 91 160 160 160 F F F 10 200-219 86 86 86 149 149 149 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 82 82 82 142 142 142 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 90 90 90 159 159 159 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 90 90 90 155 155 155 F F F 19 380-399 87 87 87 151 151 151 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 81 81 81 141 141 141 F F F 22 440-459 85 85 85 147 147 147 F F F 23 460-479 89 89 89 154 154 154 F F F 24 480-499 91 91 91 160 160 160 F F F 25 500-519 91 91 91 160 160 160 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 88 88 88 153 153 153 F F F 29 580-599 90 90 90 159 159 159 F F F Max gen: 91, Max flush: 160 Begin loop 46 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.022838417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.070525637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.07116106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.071205404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.071220202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.07123448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} {"msg":"Crucible 8c52bfa9-9ed3-4ebd-a9cc-f137c706d50b has session id: 7ef0e235-3f2f-4032-8a41-c8d20daa9160","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.071839827Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Upstairs opts: Upstairs UUID: 8c52bfa9-9ed3-4ebd-a9cc-f137c706d50b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.071886921Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Crucible stats registered with UUID: 8c52bfa9-9ed3-4ebd-a9cc-f137c706d50b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.071910757Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103256541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103295616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} {"msg":"8c52bfa9-9ed3-4ebd-a9cc-f137c706d50b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103323372Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103346269Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103391903Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103425748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103458284Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103489609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103521175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103554581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103604934Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103640639Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_connection connected from Ok(127.0.0.1:57504)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103760443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_connection connected from Ok(127.0.0.1:56465)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103797298Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_connection connected from Ok(127.0.0.1:61499)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.103828853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.104317416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.104353531Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.104482564Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.104552344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.104570652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.10458514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.104881919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105004193Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105265537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105370292Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105399088Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105413926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105528911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105551148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105564896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105730763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105749071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[0]R flush_numbers[0..12]: [156, 149, 160, 155, 160, 135, 151, 141, 157, 160, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105763499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[0]R generation[0..12]: [90, 86, 91, 90, 91, 78, 87, 81, 90, 91, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105777207Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105791045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[1]R flush_numbers[0..12]: [156, 149, 160, 155, 160, 135, 151, 141, 157, 160, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105804583Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[1]R generation[0..12]: [90, 86, 91, 90, 91, 78, 87, 81, 90, 91, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105817811Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105831569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[2]R flush_numbers[0..12]: [156, 149, 160, 155, 160, 135, 151, 141, 157, 160, 149, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105844947Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[2]R generation[0..12]: [90, 86, 91, 90, 91, 78, 87, 81, 90, 91, 86, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105858216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105871824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105885232Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.1059007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"Wait for a query_work_queue command to finish before sending IO msg":"Next flush: 161","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105919527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105932346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"":"downstairs","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105945464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105957392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"0","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.1059706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"1","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105983359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"client":"2","":"downstairs","session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.105996297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.106008475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"8c52bfa9-9ed3-4ebd-a9cc-f137c706d50b is now active with session: 7ef0e235-3f2f-4032-8a41-c8d20daa9160","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.106021083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.106032862Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550,"session_id":"7ef0e235-3f2f-4032-8a41-c8d20daa9160"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:40.10604495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 28 len 12288 data: 9 7 7 02/30 Read block 232 len 8192 03/30 Write block 420 len 8192 data: 7 8 04/30 Write block 194 len 40960 data: 5 6 5 8 7 6 6 6 5 5 05/30 Read block 274 len 20480 06/30 Write block 250 len 16384 data: 9 9 8 7 07/30 Read block 466 len 16384 08/30 Write block 199 len 20480 data: 7 7 7 6 6 09/30 Read block 213 len 16384 10/30 Write block 367 len 24576 data: 10 9 9 9 8 6 11/30 Write block 571 len 28672 data: 3 3 3 2 2 5 5 12/30 Read block 394 len 16384 13/30 Flush 14/30 Write block 391 len 20480 data: 9 7 9 8 9 15/30 Read block 432 len 8192 16/30 Read block 71 len 24576 17/30 Read block 303 len 36864 18/30 Read block 68 len 4096 19/30 Write block 508 len 32768 data: 6 6 5 5 5 7 6 7 20/30 Write block 160 len 40960 data: 8 8 5 5 7 7 7 7 8 8 21/30 Read block 246 len 4096 22/30 Write block 92 len 32768 data: 11 9 9 8 6 8 8 7 23/30 Read block 184 len 16384 24/30 Write block 462 len 8192 data: 8 7 25/30 Read block 464 len 28672 26/30 Read block 39 len 28672 27/30 Write block 444 len 40960 data: 7 9 9 7 9 11 10 8 9 10 28/30 Write block 142 len 32768 data: 7 9 10 11 9 9 9 8 29/30 Write block 479 len 40960 data: 8 6 6 7 7 6 6 8 8 10 30/30 Read block 449 len 12288 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 6 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:40.126326537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 F F F 1 020-039 92 92 92 161 161 161 F F F 2 040-059 91 91 91 160 160 160 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 91 91 91 160 160 160 T T T 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 81 81 81 141 141 141 T T T 8 160-179 90 90 90 157 157 157 T T T 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 92 92 92 161 161 161 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 90 90 90 159 159 159 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 92 92 92 161 161 161 F F F 19 380-399 87 87 87 151 151 151 T T T 20 400-419 85 85 85 147 147 147 F F F 21 420-439 92 92 92 161 161 161 F F F 22 440-459 85 85 85 147 147 147 T T T 23 460-479 89 89 89 154 154 154 T T T 24 480-499 91 91 91 160 160 160 T T T 25 500-519 91 91 91 160 160 160 T T T 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 92 92 92 161 161 161 F F F 29 580-599 90 90 90 159 159 159 F F F Max gen: 92, Max flush: 161 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 90 90 90 156 156 156 F F F 1 020-039 92 92 92 161 161 161 F F F 2 040-059 91 91 91 160 160 160 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 93 93 93 162 162 162 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 87 87 87 151 151 151 F F F 7 140-159 93 93 93 162 162 162 F F F 8 160-179 93 93 93 162 162 162 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 86 86 86 149 149 149 F F F 12 240-259 92 92 92 161 161 161 F F F 13 260-279 85 85 85 147 147 147 F F F 14 280-299 90 90 90 159 159 159 F F F 15 300-319 85 85 85 147 147 147 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 92 92 92 161 161 161 F F F 19 380-399 93 93 93 162 162 162 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 92 92 92 161 161 161 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 93 93 93 162 162 162 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 92 92 92 161 161 161 F F F 29 580-599 90 90 90 159 159 159 F F F Max gen: 93, Max flush: 162 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.422187026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.467146729Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.467770473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.467812387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.467826985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.467841603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} {"msg":"Crucible ae660a28-af8d-4d55-aa86-a7888bbec94e has session id: e99130a4-1fa1-46c7-8446-b5a4ce6596bf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.468494504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Upstairs opts: Upstairs UUID: ae660a28-af8d-4d55-aa86-a7888bbec94e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.468555376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Crucible stats registered with UUID: ae660a28-af8d-4d55-aa86-a7888bbec94e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.468575703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.500651047Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.500692772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} {"msg":"ae660a28-af8d-4d55-aa86-a7888bbec94e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.500759642Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.500801457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50085212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.500893794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.500984412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501063611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501121213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50122037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501253275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501281461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:55544)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501300069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:43518)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501317286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_connection connected from Ok(127.0.0.1:56524)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501344542Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501809709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501837085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501852213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501865921Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50187908Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.501892398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.502473928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.502502704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.502548928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503026643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503051409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503064877Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503140377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503159285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503176522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50318895Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503202339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[0]R flush_numbers[0..12]: [156, 161, 160, 155, 162, 135, 151, 162, 162, 161, 161, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503215947Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[0]R generation[0..12]: [90, 92, 91, 90, 93, 78, 87, 93, 93, 92, 92, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503229315Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503242833Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[1]R flush_numbers[0..12]: [156, 161, 160, 155, 162, 135, 151, 162, 162, 161, 161, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503256291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[1]R generation[0..12]: [90, 92, 91, 90, 93, 78, 87, 93, 93, 92, 92, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503269399Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503282997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[2]R flush_numbers[0..12]: [156, 161, 160, 155, 162, 135, 151, 162, 162, 161, 161, 149]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503296226Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[2]R generation[0..12]: [90, 92, 91, 90, 93, 78, 87, 93, 93, 92, 92, 86]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503309474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503322952Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50333541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503348289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503362807Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503383974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"":"downstairs","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503396942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50340859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"0","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503421319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"1","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503433997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"client":"2","":"downstairs","session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503446815Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503458773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"ae660a28-af8d-4d55-aa86-a7888bbec94e is now active with session: e99130a4-1fa1-46c7-8446-b5a4ce6596bf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503471222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.50348284Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558,"session_id":"e99130a4-1fa1-46c7-8446-b5a4ce6596bf"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:44.503494728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 4096 data: 6 02/30 Write block 271 len 36864 data: 12 12 10 10 8 11 11 11 7 03/30 Read block 15 len 36864 04/30 Read block 576 len 36864 05/30 Write block 8 len 32768 data: 7 7 8 8 8 8 10 10 06/30 Read block 463 len 24576 07/30 Flush 08/30 Flush 09/30 Write block 256 len 28672 data: 9 8 11 10 8 7 8 10/30 Write block 232 len 12288 data: 4 5 7 11/30 Write block 253 len 20480 data: 8 8 8 10 9 12/30 Write block 310 len 28672 data: 5 5 4 6 7 6 10 13/30 Flush 14/30 Write block 511 len 12288 data: 6 6 8 15/30 Read block 73 len 16384 16/30 Write block 385 len 32768 data: 10 13 13 15 10 9 10 8 17/30 Write block 29 len 32768 data: 8 8 5 5 6 7 7 6 18/30 Write block 367 len 12288 data: 11 10 10 19/30 Flush 20/30 Write block 146 len 24576 data: 10 10 10 9 9 7 21/30 Flush 22/30 Write block 34 len 28672 data: 8 8 7 7 6 6 6 23/30 Write block 129 len 28672 data: 15 15 14 13 11 10 11 24/30 Flush 25/30 Write block 565 len 24576 data: 7 6 7 7 7 5 26/30 Write block 303 len 20480 data: 6 7 5 6 5 27/30 Read block 157 len 4096 28/30 Read block 135 len 4096 29/30 Write block 167 len 32768 data: 8 9 9 8 7 9 9 8 30/30 Write block 125 len 8192 data: 7 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:44.521603223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 168 168 168 F F F 2 040-059 94 94 94 168 168 168 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 94 94 94 168 168 168 T T T 7 140-159 94 94 94 167 167 167 F F F 8 160-179 93 93 93 162 162 162 T T T 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 94 94 94 165 165 165 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 94 94 94 165 165 165 F F F 14 280-299 90 90 90 159 159 159 F F F 15 300-319 94 94 94 165 165 165 T T T 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 94 94 94 166 166 166 F F F 19 380-399 94 94 94 166 166 166 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 92 92 92 161 161 161 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 94 94 94 166 166 166 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 92 92 92 161 161 161 T T T 29 580-599 90 90 90 159 159 159 F F F Max gen: 94, Max flush: 168 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 168 168 168 F F F 2 040-059 94 94 94 168 168 168 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 95 95 95 169 169 169 F F F 7 140-159 94 94 94 167 167 167 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 94 94 94 165 165 165 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 94 94 94 165 165 165 F F F 14 280-299 90 90 90 159 159 159 F F F 15 300-319 95 95 95 169 169 169 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 F F F 18 360-379 94 94 94 166 166 166 F F F 19 380-399 94 94 94 166 166 166 F F F 20 400-419 85 85 85 147 147 147 F F F 21 420-439 92 92 92 161 161 161 F F F 22 440-459 93 93 93 162 162 162 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 94 94 94 166 166 166 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 95 95 95 169 169 169 F F F 29 580-599 90 90 90 159 159 159 F F F Max gen: 95, Max flush: 169 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.793963434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:48.837604446Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.83801751Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.870327322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.870880166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.870922161Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.870938998Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.870956466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} {"msg":"Crucible 81e9c4b6-5f89-4f47-8808-665a2fd9727b has session id: 5e10b2a9-e84e-4370-899a-ed090b689339","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.871576061Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Upstairs opts: Upstairs UUID: 81e9c4b6-5f89-4f47-8808-665a2fd9727b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.871627814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Crucible stats registered with UUID: 81e9c4b6-5f89-4f47-8808-665a2fd9727b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.87166271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902531349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902565024Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} {"msg":"81e9c4b6-5f89-4f47-8808-665a2fd9727b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902633545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902656162Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902694387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902727492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902761017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902793493Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902826918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902859834Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902895259Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.902927195Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_connection connected from Ok(127.0.0.1:45620)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903057977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_connection connected from Ok(127.0.0.1:43114)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903104611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_connection connected from Ok(127.0.0.1:36996)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903142095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903663624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903697389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903715317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903732035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903747952Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.903776828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.904295697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.904335002Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.904376146Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.904824505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.904846032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.90486036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.904999851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905020329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905033767Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905045845Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905063093Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[0]R flush_numbers[0..12]: [163, 168, 168, 155, 163, 135, 169, 167, 169, 161, 161, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905078001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[0]R generation[0..12]: [94, 94, 94, 90, 94, 78, 95, 94, 95, 92, 92, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905091689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905105527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[1]R flush_numbers[0..12]: [163, 168, 168, 155, 163, 135, 169, 167, 169, 161, 161, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905119215Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[1]R generation[0..12]: [94, 94, 94, 90, 94, 78, 95, 94, 95, 92, 92, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905133733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905147371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[2]R flush_numbers[0..12]: [163, 168, 168, 155, 163, 135, 169, 167, 169, 161, 161, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905161579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[2]R generation[0..12]: [94, 94, 94, 90, 94, 78, 95, 94, 95, 92, 92, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905174627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905188155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905200504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905213352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Next flush: 170","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.90522581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905238579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"":"downstairs","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:08:48.905254077Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905272964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"0","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905285832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"1","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905298371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"client":"2","":"downstairs","session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905311039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905322847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"81e9c4b6-5f89-4f47-8808-665a2fd9727b is now active with session: 5e10b2a9-e84e-4370-899a-ed090b689339","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905335275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905346954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566,"session_id":"5e10b2a9-e84e-4370-899a-ed090b689339"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:48.905358772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 447 len 28672 data: 8 10 12 11 9 10 11 02/30 Write block 141 len 20480 data: 7 8 10 11 12 03/30 Write block 505 len 8192 data: 7 7 04/30 Read block 543 len 4096 05/30 Read block 571 len 36864 06/30 Read block 532 len 32768 07/30 Read block 262 len 16384 08/30 Read block 232 len 12288 09/30 Read block 412 len 12288 10/30 Read block 387 len 4096 11/30 Write block 589 len 40960 data: 6 6 7 6 6 5 3 3 2 3 12/30 Read block 60 len 20480 13/30 Read block 528 len 24576 14/30 Flush 15/30 Write block 314 len 8192 data: 8 7 16/30 Write block 41 len 32768 data: 9 12 12 13 14 12 10 12 17/30 Read block 436 len 28672 18/30 Read block 405 len 28672 19/30 Read block 34 len 24576 20/30 Read block 90 len 40960 21/30 Read block 196 len 16384 22/30 Read block 65 len 16384 23/30 Read block 211 len 4096 24/30 Read block 307 len 4096 25/30 Write block 400 len 36864 data: 5 5 5 5 4 6 7 7 7 26/30 Read block 263 len 36864 27/30 Write block 289 len 8192 data: 13 13 28/30 Read block 589 len 24576 29/30 Write block 351 len 32768 data: 6 6 8 6 7 6 5 6 30/30 Write block 574 len 40960 data: 3 3 6 6 6 5 6 7 10 8 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 2 5 Done 15 14 14 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:48.925600724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 168 168 168 F F F 2 040-059 94 94 94 168 168 168 T T T 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 95 95 95 169 169 169 F F F 7 140-159 96 96 96 170 170 170 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 94 94 94 165 165 165 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 94 94 94 165 165 165 F F F 14 280-299 90 90 90 159 159 159 T T T 15 300-319 95 95 95 169 169 169 T T T 16 320-339 77 77 77 133 133 133 F F F 17 340-359 84 84 84 145 145 145 T T T 18 360-379 94 94 94 166 166 166 F F F 19 380-399 94 94 94 166 166 166 F F F 20 400-419 85 85 85 147 147 147 T T T 21 420-439 92 92 92 161 161 161 F F F 22 440-459 96 96 96 170 170 170 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 96 96 96 170 170 170 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 95 95 95 169 169 169 T T T 29 580-599 96 96 96 170 170 170 T T T Max gen: 96, Max flush: 170 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 168 168 168 F F F 2 040-059 97 97 97 171 171 171 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 95 95 95 169 169 169 F F F 7 140-159 96 96 96 170 170 170 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 92 92 92 161 161 161 F F F 10 200-219 92 92 92 161 161 161 F F F 11 220-239 94 94 94 165 165 165 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 94 94 94 165 165 165 F F F 14 280-299 97 97 97 171 171 171 F F F 15 300-319 97 97 97 171 171 171 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 94 94 94 166 166 166 F F F 19 380-399 94 94 94 166 166 166 F F F 20 400-419 97 97 97 171 171 171 F F F 21 420-439 92 92 92 161 161 161 F F F 22 440-459 96 96 96 170 170 170 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 96 96 96 170 170 170 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 97 97 97 171 171 171 F F F 29 580-599 97 97 97 171 171 171 F F F Max gen: 97, Max flush: 171 Begin loop 49 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.242780369Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:53.286964278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.287377982Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.319915453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.320490494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.320524899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.320538637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.320553106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} {"msg":"Crucible 25a55d95-7263-4166-8a46-601ea46e1dbf has session id: e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.321217145Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Upstairs opts: Upstairs UUID: 25a55d95-7263-4166-8a46-601ea46e1dbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.32125018Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Crucible stats registered with UUID: 25a55d95-7263-4166-8a46-601ea46e1dbf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.321271407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.352909981Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.352941047Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} {"msg":"25a55d95-7263-4166-8a46-601ea46e1dbf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353030625Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353052782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353091217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353123902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353194742Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353231347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353260253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.35328849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353314536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353339672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41126)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353366289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:65138)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353393265Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:58847)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353421561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.353995593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354027038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354053735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354079491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354138403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354167009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354630416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354760608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.354780615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355170532Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355208487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355234683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355282287Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355316072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355343238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355385213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355411629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[0]R flush_numbers[0..12]: [163, 168, 171, 155, 163, 135, 169, 170, 169, 161, 161, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355435666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[0]R generation[0..12]: [94, 94, 97, 90, 94, 78, 95, 96, 95, 92, 92, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355459433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355481999Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[1]R flush_numbers[0..12]: [163, 168, 171, 155, 163, 135, 169, 170, 169, 161, 161, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355503347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[1]R generation[0..12]: [94, 94, 97, 90, 94, 78, 95, 96, 95, 92, 92, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355526384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.35555121Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[2]R flush_numbers[0..12]: [163, 168, 171, 155, 163, 135, 169, 170, 169, 161, 161, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355577286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[2]R generation[0..12]: [94, 94, 97, 90, 94, 78, 95, 96, 95, 92, 92, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355600133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.3556256Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:08:53.355651666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355683852Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355710278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355739174Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"":"downstairs","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355766421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355791027Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"0","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355816474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"1","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.3558416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"client":"2","":"downstairs","session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355857438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355870026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 25a55d95-7263-4166-8a46-601ea46e1dbf is now active with session: e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355885714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355898113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574,"session_id":"e3f0bd58-eab0-4e9d-8881-227bd2c3e9d5"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:53.355910331Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 8192 data: 10 13 02/30 Read block 523 len 28672 03/30 Read block 251 len 36864 04/30 Read block 331 len 4096 05/30 Write block 211 len 12288 data: 9 9 9 06/30 Read block 25 len 36864 07/30 Flush 08/30 Write block 393 len 40960 data: 10 9 10 7 8 8 7 6 6 6 09/30 Read block 533 len 32768 10/30 Read block 141 len 24576 11/30 Read block 516 len 20480 12/30 Read block 23 len 4096 13/30 Read block 85 len 20480 14/30 Read block 573 len 20480 15/30 Write block 407 len 36864 data: 8 8 8 7 7 9 7 7 7 16/30 Write block 443 len 12288 data: 6 8 10 17/30 Write block 182 len 16384 data: 8 9 8 8 18/30 Read block 492 len 8192 19/30 Write block 298 len 28672 data: 12 12 10 9 8 7 8 20/30 Write block 576 len 4096 data: 7 21/30 Read block 49 len 28672 22/30 Write block 360 len 8192 data: 6 6 23/30 Write block 507 len 32768 data: 5 7 7 6 7 7 9 7 24/30 Read block 321 len 12288 25/30 Read block 437 len 4096 26/30 Read block 431 len 40960 27/30 Write block 397 len 20480 data: 9 9 8 7 7 28/30 Read block 43 len 28672 29/30 Read block 585 len 12288 30/30 Read block 332 len 24576 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 10 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 5 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 23 23 22 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:53.375140782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 168 168 168 F F F 2 040-059 98 98 98 172 172 172 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 95 95 95 169 169 169 F F F 7 140-159 96 96 96 170 170 170 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 92 92 92 161 161 161 T T T 10 200-219 98 98 98 172 172 172 F F F 11 220-239 94 94 94 165 165 165 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 94 94 94 165 165 165 F F F 14 280-299 97 97 97 171 171 171 T T T 15 300-319 97 97 97 171 171 171 T T T 16 320-339 77 77 77 133 133 133 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 94 94 94 166 166 166 T T T 19 380-399 94 94 94 166 166 166 T T T 20 400-419 97 97 97 171 171 171 T T T 21 420-439 92 92 92 161 161 161 F F F 22 440-459 96 96 96 170 170 170 T T T 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 96 96 96 170 170 170 T T T 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 97 97 97 171 171 171 T T T 29 580-599 97 97 97 171 171 171 F F F Max gen: 98, Max flush: 172 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 163 163 163 F F F 1 020-039 94 94 94 168 168 168 F F F 2 040-059 98 98 98 172 172 172 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 95 95 95 169 169 169 F F F 7 140-159 96 96 96 170 170 170 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 99 99 99 173 173 173 F F F 10 200-219 98 98 98 172 172 172 F F F 11 220-239 94 94 94 165 165 165 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 94 94 94 165 165 165 F F F 14 280-299 99 99 99 173 173 173 F F F 15 300-319 99 99 99 173 173 173 F F F 16 320-339 77 77 77 133 133 133 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 99 99 99 173 173 173 F F F 19 380-399 99 99 99 173 173 173 F F F 20 400-419 99 99 99 173 173 173 F F F 21 420-439 92 92 92 161 161 161 F F F 22 440-459 99 99 99 173 173 173 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 99 99 99 173 173 173 F F F 29 580-599 97 97 97 171 171 171 F F F Max gen: 99, Max flush: 173 Begin loop 50 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.704657878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.749364335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.749956414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.74999378Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.750034294Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.750049712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} {"msg":"Crucible 32dc735c-61c3-49f7-a677-58eb00ee11fd has session id: fdd461f3-e3ee-40ca-9916-8662a071d202","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.750471194Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Upstairs opts: Upstairs UUID: 32dc735c-61c3-49f7-a677-58eb00ee11fd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.750513688Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Crucible stats registered with UUID: 32dc735c-61c3-49f7-a677-58eb00ee11fd","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.750540355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.781802219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.781849473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} {"msg":"32dc735c-61c3-49f7-a677-58eb00ee11fd active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.781921653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.781952649Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.781994613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782049776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782075162Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782102229Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782126885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782150702Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782193276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782243399Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_connection connected from Ok(127.0.0.1:61049)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782290853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_connection connected from Ok(127.0.0.1:59402)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.782337526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_connection connected from Ok(127.0.0.1:48565)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.7823844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783247402Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783274968Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783289056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783302724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783315692Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783328521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783859928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783900692Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.783961344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784399014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.78442682Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784442129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784513529Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784533396Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784546464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784558473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.78457512Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[0]R flush_numbers[0..12]: [163, 168, 172, 155, 163, 135, 169, 170, 169, 173, 172, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784589118Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[0]R generation[0..12]: [94, 94, 98, 90, 94, 78, 95, 96, 95, 99, 98, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784602427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784616155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[1]R flush_numbers[0..12]: [163, 168, 172, 155, 163, 135, 169, 170, 169, 173, 172, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784629653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[1]R generation[0..12]: [94, 94, 98, 90, 94, 78, 95, 96, 95, 99, 98, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784642721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784656249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[2]R flush_numbers[0..12]: [163, 168, 172, 155, 163, 135, 169, 170, 169, 173, 172, 165]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784669647Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[2]R generation[0..12]: [94, 94, 98, 90, 94, 78, 95, 96, 95, 99, 98, 94]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784682776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784696264Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784708802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.78472166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784734318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784746617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"":"downstairs","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784761095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Transition from WaitQuorum to Active",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784781342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"0","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.78479535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"1","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784808419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"client":"2","":"downstairs","session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784821297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784833205Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"32dc735c-61c3-49f7-a677-58eb00ee11fd is now active with session: fdd461f3-e3ee-40ca-9916-8662a071d202","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784845783Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.784857312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582,"session_id":"fdd461f3-e3ee-40ca-9916-8662a071d202"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:08:57.78486912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 396 len 16384 data: 8 10 10 9 02/30 Write block 265 len 28672 data: 12 11 9 9 9 11 13 03/30 Read block 262 len 8192 04/30 Read block 273 len 28672 05/30 Read block 221 len 8192 06/30 Flush 07/30 Write block 121 len 16384 data: 7 8 5 5 08/30 Flush 09/30 Write block 213 len 4096 data: 10 10/30 Read block 507 len 28672 11/30 Write block 15 len 32768 data: 11 11 9 11 11 9 7 7 12/30 Read block 242 len 16384 13/30 Read block 56 len 32768 14/30 Write block 330 len 28672 data: 5 5 7 8 6 7 9 15/30 Write block 54 len 16384 data: 8 9 12 13 16/30 Read block 163 len 24576 17/30 Read block 252 len 36864 18/30 Flush 19/30 Flush 20/30 Write block 20 len 16384 data: 10 8 8 6 21/30 Flush 22/30 Write block 227 len 16384 data: 4 5 5 5 23/30 Read block 93 len 4096 24/30 Read block 514 len 32768 25/30 Write block 417 len 16384 data: 7 7 6 8 26/30 Write block 569 len 24576 data: 8 6 4 4 4 4 27/30 Read block 4 len 12288 28/30 Read block 551 len 32768 29/30 Read block 311 len 36864 30/30 Read block 256 len 36864 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 9 9 8 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:08:57.804771959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 176 176 176 F F F 1 020-039 100 100 100 178 178 178 F F F 2 040-059 100 100 100 176 176 176 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 100 100 100 175 175 175 F F F 7 140-159 96 96 96 170 170 170 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 99 99 99 173 173 173 F F F 10 200-219 100 100 100 176 176 176 F F F 11 220-239 94 94 94 165 165 165 T T T 12 240-259 94 94 94 165 165 165 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 99 99 99 173 173 173 F F F 15 300-319 99 99 99 173 173 173 F F F 16 320-339 100 100 100 176 176 176 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 99 99 99 173 173 173 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 99 99 99 173 173 173 T T T 21 420-439 92 92 92 161 161 161 T T T 22 440-459 99 99 99 173 173 173 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 99 99 99 173 173 173 T T T 29 580-599 97 97 97 171 171 171 F F F Max gen: 100, Max flush: 178 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 176 176 176 F F F 1 020-039 100 100 100 178 178 178 F F F 2 040-059 100 100 100 176 176 176 F F F 3 060-079 90 90 90 155 155 155 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 100 100 100 175 175 175 F F F 7 140-159 96 96 96 170 170 170 F F F 8 160-179 95 95 95 169 169 169 F F F 9 180-199 99 99 99 173 173 173 F F F 10 200-219 100 100 100 176 176 176 F F F 11 220-239 101 101 101 179 179 179 F F F 12 240-259 94 94 94 165 165 165 F F F 13 260-279 100 100 100 174 174 174 F F F 14 280-299 99 99 99 173 173 173 F F F 15 300-319 99 99 99 173 173 173 F F F 16 320-339 100 100 100 176 176 176 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 99 99 99 173 173 173 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 101 101 101 179 179 179 F F F 21 420-439 101 101 101 179 179 179 F F F 22 440-459 99 99 99 173 173 173 F F F 23 460-479 93 93 93 162 162 162 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 101 101 101 179 179 179 F F F 29 580-599 97 97 97 171 171 171 F F F Max gen: 101, Max flush: 179 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.074052202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:02.119611762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.11999519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.152570246Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.153121771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.153162925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.153180432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.15319771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} {"msg":"Crucible 33034c40-e0c5-4dbe-a6a2-111906017f90 has session id: cca0628a-9b56-450d-8dce-18da71c959e0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.153800708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Upstairs opts: Upstairs UUID: 33034c40-e0c5-4dbe-a6a2-111906017f90, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.153846041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Crucible stats registered with UUID: 33034c40-e0c5-4dbe-a6a2-111906017f90","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.153873948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185096199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185126875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} {"msg":"33034c40-e0c5-4dbe-a6a2-111906017f90 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185201504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185223491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185251407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185268425Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185284733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185300581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185326897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185345525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185436132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185471917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:62375)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185504313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46030)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185537328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_connection connected from Ok(127.0.0.1:34111)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.185571464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.186409339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.186436735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.186454163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.18647593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.186502186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.186528543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187017406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187118062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187139839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187556782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187584539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187599656Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187686665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187707531Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.18772109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187733058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187746036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[0]R flush_numbers[0..12]: [176, 178, 176, 155, 163, 135, 175, 170, 169, 173, 176, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187759544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[0]R generation[0..12]: [100, 100, 100, 90, 94, 78, 100, 96, 95, 99, 100, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187777222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.18779216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[1]R flush_numbers[0..12]: [176, 178, 176, 155, 163, 135, 175, 170, 169, 173, 176, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187806038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[1]R generation[0..12]: [100, 100, 100, 90, 94, 78, 100, 96, 95, 99, 100, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187819416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187832965Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[2]R flush_numbers[0..12]: [176, 178, 176, 155, 163, 135, 175, 170, 169, 173, 176, 179]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187846543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[2]R generation[0..12]: [100, 100, 100, 90, 94, 78, 100, 96, 95, 99, 100, 101]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187859861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187873449Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187886107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187898916Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187911594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187930601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"":"downstairs","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187943739Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187955638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"0","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187971106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"1","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187984094Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"client":"2","":"downstairs","session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.187996882Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.18800911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"33034c40-e0c5-4dbe-a6a2-111906017f90 is now active with session: cca0628a-9b56-450d-8dce-18da71c959e0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.188021789Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.188033577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590,"session_id":"cca0628a-9b56-450d-8dce-18da71c959e0"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:02.188045495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 43 len 20480 data: 13 14 15 13 11 02/30 Write block 231 len 40960 data: 5 5 6 8 9 9 10 10 9 9 03/30 Read block 501 len 24576 04/30 Read block 178 len 28672 05/30 Write block 580 len 28672 data: 7 8 11 9 7 7 6 06/30 Write block 250 len 32768 data: 10 10 9 9 9 9 11 10 07/30 Read block 348 len 4096 08/30 Read block 15 len 36864 09/30 Write block 144 len 28672 data: 12 13 11 11 11 10 10 10/30 Read block 316 len 16384 11/30 Write block 266 len 24576 data: 12 10 10 10 12 14 12/30 Read block 571 len 4096 13/30 Write block 171 len 40960 data: 8 10 10 9 7 5 6 6 5 5 14/30 Read block 545 len 16384 15/30 Write block 63 len 8192 data: 10 9 16/30 Write block 572 len 32768 data: 5 5 5 4 8 7 7 6 17/30 Read block 40 len 8192 18/30 Read block 131 len 40960 19/30 Read block 490 len 8192 20/30 Write block 407 len 32768 data: 9 9 9 8 8 10 8 8 21/30 Read block 302 len 4096 22/30 Write block 236 len 36864 data: 10 11 11 10 10 11 10 10 10 23/30 Read block 336 len 8192 24/30 Read block 479 len 12288 25/30 Read block 286 len 16384 26/30 Read block 369 len 20480 27/30 Write block 560 len 28672 data: 9 10 9 8 7 8 7 28/30 Read block 319 len 28672 29/30 Write block 32 len 36864 data: 6 7 9 9 8 8 7 7 7 30/30 Write block 473 len 24576 data: 6 9 8 8 9 9 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 6 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 1 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:02.209306288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 176 176 176 F F F 1 020-039 100 100 100 178 178 178 T T T 2 040-059 100 100 100 176 176 176 T T T 3 060-079 90 90 90 155 155 155 T T T 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 100 100 100 175 175 175 F F F 7 140-159 96 96 96 170 170 170 T T T 8 160-179 95 95 95 169 169 169 T T T 9 180-199 99 99 99 173 173 173 T T T 10 200-219 100 100 100 176 176 176 F F F 11 220-239 101 101 101 179 179 179 T T T 12 240-259 94 94 94 165 165 165 T T T 13 260-279 100 100 100 174 174 174 T T T 14 280-299 99 99 99 173 173 173 F F F 15 300-319 99 99 99 173 173 173 F F F 16 320-339 100 100 100 176 176 176 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 99 99 99 173 173 173 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 101 101 101 179 179 179 T T T 21 420-439 101 101 101 179 179 179 F F F 22 440-459 99 99 99 173 173 173 F F F 23 460-479 93 93 93 162 162 162 T T T 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 101 101 101 179 179 179 T T T 29 580-599 97 97 97 171 171 171 T T T Max gen: 101, Max flush: 179 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 100 100 100 176 176 176 F F F 1 020-039 102 102 102 180 180 180 F F F 2 040-059 102 102 102 180 180 180 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 94 94 94 163 163 163 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 100 100 100 175 175 175 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 102 102 102 180 180 180 F F F 9 180-199 102 102 102 180 180 180 F F F 10 200-219 100 100 100 176 176 176 F F F 11 220-239 102 102 102 180 180 180 F F F 12 240-259 102 102 102 180 180 180 F F F 13 260-279 102 102 102 180 180 180 F F F 14 280-299 99 99 99 173 173 173 F F F 15 300-319 99 99 99 173 173 173 F F F 16 320-339 100 100 100 176 176 176 F F F 17 340-359 97 97 97 171 171 171 F F F 18 360-379 99 99 99 173 173 173 F F F 19 380-399 100 100 100 174 174 174 F F F 20 400-419 102 102 102 180 180 180 F F F 21 420-439 101 101 101 179 179 179 F F F 22 440-459 99 99 99 173 173 173 F F F 23 460-479 102 102 102 180 180 180 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 91 91 91 160 160 160 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 102 102 102 180 180 180 F F F Max gen: 102, Max flush: 180 Begin loop 52 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.570263237Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.615044674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.615686556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.615727131Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.615742378Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.615756507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} {"msg":"Crucible 39955a44-9f8f-4cbd-b845-58d45c0fc4c1 has session id: ff04ebc6-acb6-4408-8044-02a5e6e28db5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.616359124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Upstairs opts: Upstairs UUID: 39955a44-9f8f-4cbd-b845-58d45c0fc4c1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.61639276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Crucible stats registered with UUID: 39955a44-9f8f-4cbd-b845-58d45c0fc4c1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.616412847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648005897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648037723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} {"msg":"39955a44-9f8f-4cbd-b845-58d45c0fc4c1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648108583Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.64813091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648159336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648177644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648194481Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648210699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648227057Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648243165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648259872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.64827688Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_connection connected from Ok(127.0.0.1:32779)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648357569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_connection connected from Ok(127.0.0.1:35082)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648405912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37077)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.648531025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649194544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649232959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649262065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649290451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649317388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649345304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649844786Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649873252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.649970898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650445044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.65047318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650490007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650561798Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650580875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650593953Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650606031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650623099Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[0]R flush_numbers[0..12]: [176, 180, 180, 180, 163, 135, 175, 180, 180, 180, 176, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650637137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[0]R generation[0..12]: [100, 102, 102, 102, 94, 78, 100, 102, 102, 102, 100, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650650805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650664424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[1]R flush_numbers[0..12]: [176, 180, 180, 180, 163, 135, 175, 180, 180, 180, 176, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650677822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[1]R generation[0..12]: [100, 102, 102, 102, 94, 78, 100, 102, 102, 102, 100, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.65069124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650705038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[2]R flush_numbers[0..12]: [176, 180, 180, 180, 163, 135, 175, 180, 180, 180, 176, 180]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650718596Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[2]R generation[0..12]: [100, 102, 102, 102, 94, 78, 100, 102, 102, 102, 100, 102]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650731894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650745472Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Max found gen is 103","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650757841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Generation requested: 104 >= found:103","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650770859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650783477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7",Wait for a query_work_queue command to finish before sending IO "pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650803205Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"":"downstairs","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650818073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650829951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"0","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650842809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"1","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650855597Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"client":"2","":"downstairs","session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650868496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650880434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"39955a44-9f8f-4cbd-b845-58d45c0fc4c1 is now active with session: ff04ebc6-acb6-4408-8044-02a5e6e28db5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650893012Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650904901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598,"session_id":"ff04ebc6-acb6-4408-8044-02a5e6e28db5"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:06.650920359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 162 len 8192 data: 6 6 02/30 Write block 354 len 24576 data: 7 8 7 6 7 6 03/30 Read block 236 len 32768 04/30 Read block 531 len 36864 05/30 Read block 416 len 12288 06/30 Read block 8 len 32768 07/30 Write block 385 len 40960 data: 11 14 14 16 11 10 11 9 11 10 08/30 Flush 09/30 Write block 545 len 4096 data: 9 10/30 Write block 230 len 40960 data: 6 6 6 7 9 10 11 12 12 11 11/30 Write block 216 len 28672 data: 10 9 8 8 8 7 10 12/30 Write block 131 len 4096 data: 15 13/30 Write block 336 len 16384 data: 10 8 10 14 14/30 Write block 6 len 4096 data: 8 15/30 Write block 246 len 28672 data: 10 9 9 9 11 11 10 16/30 Flush 17/30 Write block 291 len 40960 data: 14 14 13 10 9 10 11 13 13 11 18/30 Write block 83 len 8192 data: 5 6 19/30 Read block 584 len 36864 20/30 Read block 116 len 8192 21/30 Read block 185 len 28672 22/30 Read block 503 len 36864 23/30 Write block 446 len 16384 data: 10 9 11 13 24/30 Read block 202 len 12288 25/30 Read block 93 len 20480 26/30 Write block 369 len 40960 data: 11 10 9 7 6 8 10 9 10 11 27/30 Write block 17 len 32768 data: 10 12 12 11 9 9 7 6 28/30 Read block 153 len 40960 29/30 Read block 101 len 24576 30/30 Write block 234 len 8192 data: 10 11 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 10 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:06.669954286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1598} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 182 182 182 T T T 1 020-039 102 102 102 180 180 180 T T T 2 040-059 102 102 102 180 180 180 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 94 94 94 163 163 163 T T T 5 100-119 78 78 78 135 135 135 F F F 6 120-139 104 104 104 182 182 182 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 104 104 104 181 181 181 F F F 9 180-199 102 102 102 180 180 180 F F F 10 200-219 104 104 104 182 182 182 F F F 11 220-239 104 104 104 182 182 182 T T T 12 240-259 104 104 104 182 182 182 F F F 13 260-279 102 102 102 180 180 180 F F F 14 280-299 99 99 99 173 173 173 T T T 15 300-319 99 99 99 173 173 173 T T T 16 320-339 104 104 104 182 182 182 F F F 17 340-359 104 104 104 181 181 181 F F F 18 360-379 99 99 99 173 173 173 T T T 19 380-399 104 104 104 181 181 181 F F F 20 400-419 102 102 102 180 180 180 F F F 21 420-439 101 101 101 179 179 179 F F F 22 440-459 99 99 99 173 173 173 T T T 23 460-479 102 102 102 180 180 180 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 104 104 104 182 182 182 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 102 102 102 180 180 180 F F F Max gen: 104, Max flush: 182 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 183 183 183 F F F 1 020-039 105 105 105 183 183 183 F F F 2 040-059 102 102 102 180 180 180 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 104 104 104 182 182 182 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 104 104 104 181 181 181 F F F 9 180-199 102 102 102 180 180 180 F F F 10 200-219 104 104 104 182 182 182 F F F 11 220-239 105 105 105 183 183 183 F F F 12 240-259 104 104 104 182 182 182 F F F 13 260-279 102 102 102 180 180 180 F F F 14 280-299 105 105 105 183 183 183 F F F 15 300-319 105 105 105 183 183 183 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 104 104 104 181 181 181 F F F 18 360-379 105 105 105 183 183 183 F F F 19 380-399 104 104 104 181 181 181 F F F 20 400-419 102 102 102 180 180 180 F F F 21 420-439 101 101 101 179 179 179 F F F 22 440-459 105 105 105 183 183 183 F F F 23 460-479 102 102 102 180 180 180 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 99 99 99 173 173 173 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 104 104 104 182 182 182 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 102 102 102 180 180 180 F F F Max gen: 105, Max flush: 183 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:10.964431204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.009483673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.010197596Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.01024008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.010257598Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.010285984Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} {"msg":"Crucible 1c3021fd-4dbc-4a61-8a90-e4951dcbbc36 has session id: a013f361-f78e-430a-994c-eb3a53a15fb1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.010862985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Upstairs opts: Upstairs UUID: 1c3021fd-4dbc-4a61-8a90-e4951dcbbc36, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.01089658Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Crucible stats registered with UUID: 1c3021fd-4dbc-4a61-8a90-e4951dcbbc36","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.010911288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042481842Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042517197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} {"msg":"1c3021fd-4dbc-4a61-8a90-e4951dcbbc36 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042587267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042609844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.04263823Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042655508Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042672236Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042688504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042705141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042720989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042737537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042753904Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54152)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042876368Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:46392)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042926841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63602)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.042964606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.043686517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.043725252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.043816209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.043843286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.043860883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.043877991Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044277736Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044417987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044458031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044785787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044812263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044826641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.044976691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045025314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045043051Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.04505542Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045068758Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[0]R flush_numbers[0..12]: [183, 183, 180, 180, 183, 135, 182, 180, 181, 180, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045086496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[0]R generation[0..12]: [105, 105, 102, 102, 105, 78, 104, 102, 104, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045100974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045118271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[1]R flush_numbers[0..12]: [183, 183, 180, 180, 183, 135, 182, 180, 181, 180, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045142848Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[1]R generation[0..12]: [105, 105, 102, 102, 105, 78, 104, 102, 104, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045169654Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.04519727Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[2]R flush_numbers[0..12]: [183, 183, 180, 180, 183, 135, 182, 180, 181, 180, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045222677Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":Wait for a query_work_queue command to finish before sending IO 1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[2]R generation[0..12]: [105, 105, 102, 102, 105, 78, 104, 102, 104, 102, 104, 105]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045254933Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045282399Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045299376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045312795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045328252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.0453507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"":"downstairs","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045376416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045400822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"0","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.04541562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"1","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045437517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"client":"2","":"downstairs","session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045462884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045486851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"1c3021fd-4dbc-4a61-8a90-e4951dcbbc36 is now active with session: a013f361-f78e-430a-994c-eb3a53a15fb1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045502439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:11.045516887Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606,"session_id":"a013f361-f78e-430a-994c-eb3a53a15fb1"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:09:11.045538994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 226 len 32768 data: 8 5 6 6 7 7 7 8 02/30 Write block 587 len 36864 data: 7 7 7 7 8 7 7 6 4 03/30 Read block 308 len 4096 04/30 Read block 21 len 4096 05/30 Read block 292 len 32768 06/30 Write block 507 len 32768 data: 6 8 8 7 8 8 10 8 07/30 Read block 359 len 40960 08/30 Write block 435 len 12288 data: 7 8 4 09/30 Flush 10/30 Write block 377 len 40960 data: 11 12 12 12 12 10 9 10 12 15 11/30 Read block 228 len 12288 12/30 Flush 13/30 Write block 182 len 40960 data: 9 10 9 9 8 9 8 6 6 5 14/30 Read block 163 len 16384 15/30 Read block 4 len 36864 16/30 Write block 422 len 20480 data: 8 12 8 9 9 17/30 Read block 487 len 32768 18/30 Write block 424 len 8192 data: 9 10 19/30 Read block 218 len 24576 20/30 Write block 453 len 36864 data: 12 7 6 5 6 7 5 6 6 21/30 Flush 22/30 Write block 452 len 8192 data: 11 13 23/30 Read block 285 len 12288 24/30 Write block 128 len 8192 data: 14 16 25/30 Read block 101 len 40960 26/30 Read block 545 len 16384 27/30 Write block 358 len 24576 data: 8 7 7 7 8 8 28/30 Write block 237 len 20480 data: 13 13 12 11 12 29/30 Write block 456 len 36864 data: 6 7 8 6 7 7 9 8 5 30/30 Read block 355 len 20480 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 2 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 9 8 7 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:11.066373525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 183 183 183 F F F 1 020-039 105 105 105 183 183 183 F F F 2 040-059 102 102 102 180 180 180 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 104 104 104 182 182 182 T T T 7 140-159 102 102 102 180 180 180 F F F 8 160-179 104 104 104 181 181 181 F F F 9 180-199 106 106 106 186 186 186 F F F 10 200-219 104 104 104 182 182 182 F F F 11 220-239 106 106 106 184 184 184 T T T 12 240-259 104 104 104 182 182 182 T T T 13 260-279 102 102 102 180 180 180 F F F 14 280-299 105 105 105 183 183 183 F F F 15 300-319 105 105 105 183 183 183 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 104 104 104 181 181 181 T T T 18 360-379 106 106 106 185 185 185 T T T 19 380-399 106 106 106 185 185 185 F F F 20 400-419 102 102 102 180 180 180 F F F 21 420-439 106 106 106 186 186 186 F F F 22 440-459 106 106 106 186 186 186 T T T 23 460-479 106 106 106 186 186 186 T T T 24 480-499 93 93 93 162 162 162 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 104 104 104 182 182 182 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 106 106 106 184 184 184 F F F Max gen: 106, Max flush: 186 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 105 105 105 183 183 183 F F F 1 020-039 105 105 105 183 183 183 F F F 2 040-059 102 102 102 180 180 180 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 107 107 107 187 187 187 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 104 104 104 181 181 181 F F F 9 180-199 106 106 106 186 186 186 F F F 10 200-219 104 104 104 182 182 182 F F F 11 220-239 107 107 107 187 187 187 F F F 12 240-259 107 107 107 187 187 187 F F F 13 260-279 102 102 102 180 180 180 F F F 14 280-299 105 105 105 183 183 183 F F F 15 300-319 105 105 105 183 183 183 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 107 107 107 187 187 187 F F F 18 360-379 107 107 107 187 187 187 F F F 19 380-399 106 106 106 185 185 185 F F F 20 400-419 102 102 102 180 180 180 F F F 21 420-439 106 106 106 186 186 186 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 107 107 107 187 187 187 F F F 24 480-499 93 93 93 162 162 162 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 91 91 91 160 160 160 F F F 27 540-559 104 104 104 182 182 182 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 106 106 106 184 184 184 F F F Max gen: 107, Max flush: 187 Begin loop 54 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.358600801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.403022728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.403667279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.403709414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.403724092Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.40373822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} {"msg":"Crucible a6799286-74c9-48cf-9cec-f5160e0557eb has session id: e2368da3-f822-43b8-8f44-4cf3300af235","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.404297063Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Upstairs opts: Upstairs UUID: a6799286-74c9-48cf-9cec-f5160e0557eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.404338757Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Crucible stats registered with UUID: a6799286-74c9-48cf-9cec-f5160e0557eb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.404361554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435314242Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435347058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} {"msg":"a6799286-74c9-48cf-9cec-f5160e0557eb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.43540569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435434326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435462562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.43547776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435491238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435504126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435517185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435529993Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435543551Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435556839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_connection connected from Ok(127.0.0.1:34566)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435702419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_connection connected from Ok(127.0.0.1:46500)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435750282Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_connection connected from Ok(127.0.0.1:34351)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.435857678Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.436389195Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.43642535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.436448127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.436463625Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.436640881Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.436679515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437050295Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437090129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437362832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437622506Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437649813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437663981Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.43774637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437765747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437779285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437972009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.437990896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[0]R flush_numbers[0..12]: [183, 183, 180, 180, 183, 135, 187, 180, 181, 186, 182, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438004744Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[0]R generation[0..12]: [105, 105, 102, 102, 105, 78, 107, 102, 104, 106, 104, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438018392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438033101Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[1]R flush_numbers[0..12]: [183, 183, 180, 180, 183, 135, 187, 180, 181, 186, 182, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438050628Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[1]R generation[0..12]: [105, 105, 102, 102, 105, 78, 107, 102, 104, 106, 104, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438065006Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438078794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[2]R flush_numbers[0..12]: [183, 183, 180, 180, 183, 135, 187, 180, 181, 186, 182, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438092292Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[2]R generation[0..12]: [105, 105, 102, 102, 105, 78, 107, 102, 104, 106, 104, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438106531Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438120158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438132747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438145695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438158413Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438171102Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438190489Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438202327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"0","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438215216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"1","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438227914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"client":"2","":"downstairs","session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438243592Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.43825582Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"a6799286-74c9-48cf-9cec-f5160e0557eb is now active with session: e2368da3-f822-43b8-8f44-4cf3300af235","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438268638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438280467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614,"session_id":"e2368da3-f822-43b8-8f44-4cf3300af235"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:15.438292375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 539 len 8192 data: 7 8 02/30 Read block 584 len 16384 03/30 Read block 25 len 8192 04/30 Write block 270 len 32768 data: 13 15 13 11 11 9 12 12 05/30 Write block 40 len 12288 data: 8 11 14 06/30 Read block 199 len 20480 07/30 Write block 293 len 40960 data: 14 11 10 11 12 14 14 12 10 9 08/30 Write block 3 len 8192 data: 6 9 09/30 Read block 86 len 32768 10/30 Read block 423 len 12288 11/30 Write block 23 len 32768 data: 8 7 6 6 7 10 9 9 12/30 Flush 13/30 Write block 285 len 36864 data: 9 10 12 13 14 14 15 15 15 14/30 Write block 485 len 36864 data: 7 9 9 11 8 9 9 10 12 15/30 Write block 206 len 12288 data: 8 9 10 16/30 Write block 35 len 28672 data: 10 9 9 8 8 9 12 17/30 Read block 216 len 12288 18/30 Read block 173 len 36864 19/30 Read block 522 len 40960 20/30 Read block 100 len 8192 21/30 Read block 136 len 8192 22/30 Read block 133 len 20480 23/30 Write block 467 len 40960 data: 5 5 6 6 6 6 7 10 9 9 24/30 Read block 515 len 32768 25/30 Write block 378 len 36864 data: 13 13 13 13 11 10 11 13 16 26/30 Flush 27/30 Write block 416 len 40960 data: 7 8 8 7 9 9 9 13 10 11 28/30 Read block 58 len 32768 29/30 Write block 177 len 16384 data: 7 7 6 6 30/30 Read block 330 len 16384 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:15.458989505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 188 188 188 F F F 1 020-039 108 108 108 189 189 189 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 107 107 107 187 187 187 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 104 104 104 181 181 181 T T T 9 180-199 106 106 106 186 186 186 T T T 10 200-219 108 108 108 189 189 189 F F F 11 220-239 107 107 107 187 187 187 F F F 12 240-259 107 107 107 187 187 187 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 108 108 108 189 189 189 F F F 15 300-319 108 108 108 188 188 188 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 107 107 107 187 187 187 F F F 18 360-379 108 108 108 189 189 189 F F F 19 380-399 108 108 108 189 189 189 F F F 20 400-419 102 102 102 180 180 180 T T T 21 420-439 106 106 106 186 186 186 T T T 22 440-459 107 107 107 187 187 187 F F F 23 460-479 108 108 108 189 189 189 F F F 24 480-499 108 108 108 189 189 189 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 108 108 108 188 188 188 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 106 106 106 184 184 184 F F F Max gen: 108, Max flush: 189 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 188 188 188 F F F 1 020-039 108 108 108 189 189 189 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 107 107 107 187 187 187 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 109 109 109 190 190 190 F F F 9 180-199 109 109 109 190 190 190 F F F 10 200-219 108 108 108 189 189 189 F F F 11 220-239 107 107 107 187 187 187 F F F 12 240-259 107 107 107 187 187 187 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 108 108 108 189 189 189 F F F 15 300-319 108 108 108 188 188 188 F F F 16 320-339 104 104 104 182 182 182 F F F 17 340-359 107 107 107 187 187 187 F F F 18 360-379 108 108 108 189 189 189 F F F 19 380-399 108 108 108 189 189 189 F F F 20 400-419 109 109 109 190 190 190 F F F 21 420-439 109 109 109 190 190 190 F F F 22 440-459 107 107 107 187 187 187 F F F 23 460-479 108 108 108 189 189 189 F F F 24 480-499 108 108 108 189 189 189 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 108 108 108 188 188 188 F F F 28 560-579 102 102 102 180 180 180 F F F 29 580-599 106 106 106 184 184 184 F F F Max gen: 109, Max flush: 190 Begin loop 55 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.726236616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.772507168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773170648Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773215912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773231109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773246547Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} {"msg":"Crucible 8854980c-b2f6-4b09-970f-46b37f0856ac has session id: 1ca80da8-9c0f-4868-be46-379aa1ddf46f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773780744Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Upstairs opts: Upstairs UUID: 8854980c-b2f6-4b09-970f-46b37f0856ac, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773825118Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Crucible stats registered with UUID: 8854980c-b2f6-4b09-970f-46b37f0856ac","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.773847085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80496852Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805010054Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} {"msg":"8854980c-b2f6-4b09-970f-46b37f0856ac active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805048149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805067187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805094693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80511157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805129688Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805158554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805196009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805228105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.8052602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805289876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_connection connected from Ok(127.0.0.1:47724)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805320122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_connection connected from Ok(127.0.0.1:57304)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805352238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_connection connected from Ok(127.0.0.1:35977)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.805385233Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.806025956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.806054072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80606896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.806090647Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.806114843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80613948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80665398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.806815997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.806881168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807206514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807246009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807265846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80737876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807405177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807419775Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807432653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807446011Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[0]R flush_numbers[0..12]: [188, 189, 189, 180, 183, 135, 187, 180, 190, 190, 189, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80745956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[0]R generation[0..12]: [108, 108, 108, 102, 105, 78, 107, 102, 109, 109, 108, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807473458Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807487156Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[1]R flush_numbers[0..12]: [188, 189, 189, 180, 183, 135, 187, 180, 190, 190, 189, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807500564Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[1]R generation[0..12]: [108, 108, 108, 102, 105, 78, 107, 102, 109, 109, 108, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807513812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80752746Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[2]R flush_numbers[0..12]: [188, 189, 189, 180, 183, 135, 187, 180, 190, 190, 189, 187]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807544048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[2]R generation[0..12]: [108, 108, 108, 102, 105, 78, 107, 102, 109, 109, 108, 107]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807557796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807571394Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807583843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807596641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Next flush: 191","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807609019Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2025-01-11T06:09:19.807621777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"":"downstairs","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807644844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807656853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"0","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807670071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"1","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807683099Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"client":"2","":"downstairs","session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807696207Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807708215Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"8854980c-b2f6-4b09-970f-46b37f0856ac is now active with session: 1ca80da8-9c0f-4868-be46-379aa1ddf46f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807720844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.807732632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622,"session_id":"1ca80da8-9c0f-4868-be46-379aa1ddf46f"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:19.80774718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 173 len 40960 data: 11 10 8 6 8 8 7 7 6 10 02/30 Read block 337 len 32768 03/30 Write block 333 len 36864 data: 9 7 8 11 9 11 15 17 14 04/30 Read block 161 len 12288 05/30 Read block 11 len 20480 06/30 Read block 296 len 28672 07/30 Flush 08/30 Write block 582 len 4096 data: 12 09/30 Write block 392 len 8192 data: 10 12 10/30 Write block 166 len 32768 data: 8 9 10 10 9 9 11 12 11/30 Write block 491 len 8192 data: 10 11 12/30 Read block 382 len 12288 13/30 Read block 343 len 40960 14/30 Write block 126 len 20480 data: 10 12 15 17 16 15/30 Read block 89 len 20480 16/30 Write block 205 len 24576 data: 7 9 10 11 10 10 17/30 Write block 314 len 36864 data: 9 8 11 12 12 12 11 10 10 18/30 Read block 491 len 36864 19/30 Read block 287 len 28672 20/30 Write block 442 len 24576 data: 4 7 9 11 11 10 21/30 Flush 22/30 Write block 216 len 40960 data: 11 10 9 9 9 8 11 10 11 10 23/30 Read block 539 len 40960 24/30 Write block 190 len 4096 data: 7 25/30 Write block 253 len 24576 data: 10 10 10 12 11 12 26/30 Read block 171 len 4096 27/30 Read block 70 len 32768 28/30 Write block 571 len 20480 data: 5 6 6 6 5 29/30 Read block 189 len 8192 30/30 Write block 360 len 8192 data: 8 8 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:19.828308279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 188 188 188 F F F 1 020-039 108 108 108 189 189 189 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 110 110 110 192 192 192 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 110 110 110 192 192 192 F F F 9 180-199 110 110 110 191 191 191 T T T 10 200-219 110 110 110 192 192 192 T T T 11 220-239 107 107 107 187 187 187 T T T 12 240-259 107 107 107 187 187 187 T T T 13 260-279 108 108 108 188 188 188 F F F 14 280-299 108 108 108 189 189 189 F F F 15 300-319 110 110 110 192 192 192 F F F 16 320-339 110 110 110 192 192 192 F F F 17 340-359 110 110 110 191 191 191 F F F 18 360-379 108 108 108 189 189 189 T T T 19 380-399 110 110 110 192 192 192 F F F 20 400-419 109 109 109 190 190 190 F F F 21 420-439 109 109 109 190 190 190 F F F 22 440-459 110 110 110 192 192 192 F F F 23 460-479 108 108 108 189 189 189 F F F 24 480-499 110 110 110 192 192 192 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 108 108 108 188 188 188 F F F 28 560-579 102 102 102 180 180 180 T T T 29 580-599 110 110 110 192 192 192 F F F Max gen: 110, Max flush: 192 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 188 188 188 F F F 1 020-039 108 108 108 189 189 189 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 F F F 6 120-139 110 110 110 192 192 192 F F F 7 140-159 102 102 102 180 180 180 F F F 8 160-179 110 110 110 192 192 192 F F F 9 180-199 111 111 111 193 193 193 F F F 10 200-219 111 111 111 193 193 193 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 108 108 108 189 189 189 F F F 15 300-319 110 110 110 192 192 192 F F F 16 320-339 110 110 110 192 192 192 F F F 17 340-359 110 110 110 191 191 191 F F F 18 360-379 111 111 111 193 193 193 F F F 19 380-399 110 110 110 192 192 192 F F F 20 400-419 109 109 109 190 190 190 F F F 21 420-439 109 109 109 190 190 190 F F F 22 440-459 110 110 110 192 192 192 F F F 23 460-479 108 108 108 189 189 189 F F F 24 480-499 110 110 110 192 192 192 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 108 108 108 188 188 188 F F F 28 560-579 111 111 111 193 193 193 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 111, Max flush: 193 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.110658584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:24.156067756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.156470331Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.188397895Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.188951179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.188986814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.189004832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.18901957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} {"msg":"Crucible 22f84d14-cc06-4411-988b-d36e387aeb54 has session id: 97b872d1-4b23-4e53-ae08-e0edeb422dd5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.189735282Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Upstairs opts: Upstairs UUID: 22f84d14-cc06-4411-988b-d36e387aeb54, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.189779526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Crucible stats registered with UUID: 22f84d14-cc06-4411-988b-d36e387aeb54","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.189802123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221398983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221430009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} {"msg":"22f84d14-cc06-4411-988b-d36e387aeb54 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221501809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221523526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221551132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.22156805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221584637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221600665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221617033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.22163665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221653828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221723489Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:45110)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221755244Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:62459)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.221989702Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41861)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.222038955Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.222568493Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.222607308Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.222636354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.22266307Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.222688346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.222717222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223250969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223289744Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.22332141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223816042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223840689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223854607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223915249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223945035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.223960203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224015725Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224034402Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[0]R flush_numbers[0..12]: [188, 189, 189, 180, 183, 135, 192, 180, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224048531Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[0]R generation[0..12]: [108, 108, 108, 102, 105, 78, 110, 102, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224062379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224076227Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[1]R flush_numbers[0..12]: [188, 189, 189, 180, 183, 135, 192, 180, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224089925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[1]R generation[0..12]: [108, 108, 108, 102, 105, 78, 110, 102, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224103843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224117841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[2]R flush_numbers[0..12]: [188, 189, 189, 180, 183, 135, 192, 180, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224131349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[2]R generation[0..12]: [108, 108, 108, 102, 105, 78, 110, 102, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224147937Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224161915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224174783Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224187561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Next flush: 194","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.22420013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:09:24.224212798Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"":"downstairs","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224229606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224241754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"0","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224255022Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"1","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224279439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"client":"2","":"downstairs","session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224294607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224306985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"22f84d14-cc06-4411-988b-d36e387aeb54 is now active with session: 97b872d1-4b23-4e53-ae08-e0edeb422dd5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224319594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.224331322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630,"session_id":"97b872d1-4b23-4e53-ae08-e0edeb422dd5"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:24.22434436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 544 len 16384 data: 10 10 9 9 02/30 Write block 8 len 40960 data: 8 8 9 9 9 9 11 12 12 11 03/30 Write block 450 len 36864 data: 12 10 12 14 8 7 7 8 9 04/30 Read block 554 len 16384 05/30 Read block 264 len 24576 06/30 Read block 359 len 4096 07/30 Write block 116 len 8192 data: 6 6 08/30 Read block 319 len 20480 09/30 Read block 432 len 36864 10/30 Write block 562 len 20480 data: 10 9 8 9 8 11/30 Read block 102 len 28672 12/30 Write block 356 len 40960 data: 8 7 9 8 9 9 9 9 8 8 13/30 Read block 477 len 4096 14/30 Write block 104 len 12288 data: 9 9 8 15/30 Write block 374 len 32768 data: 9 11 10 12 14 14 14 14 16/30 Read block 273 len 12288 17/30 Read block 470 len 20480 18/30 Write block 7 len 40960 data: 7 9 9 10 10 10 10 12 13 13 19/30 Read block 185 len 32768 20/30 Read block 324 len 20480 21/30 Read block 23 len 40960 22/30 Read block 566 len 8192 23/30 Read block 278 len 12288 24/30 Write block 9 len 12288 data: 10 11 11 25/30 Read block 409 len 32768 26/30 Write block 413 len 28672 data: 9 9 8 8 9 9 8 27/30 Write block 382 len 8192 data: 12 11 28/30 Read block 390 len 32768 29/30 Write block 349 len 28672 data: 8 6 7 7 9 8 9 30/30 Write block 157 len 12288 data: 10 8 7 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 9 Done Done Done false 1003 Acked Read 4 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 5 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 5 Done Done Done false 1010 Acked Read 7 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 1 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Write 10 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:24.244151682Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 188 188 188 T T T 1 020-039 108 108 108 189 189 189 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 78 78 78 135 135 135 T T T 6 120-139 110 110 110 192 192 192 F F F 7 140-159 102 102 102 180 180 180 T T T 8 160-179 110 110 110 192 192 192 F F F 9 180-199 111 111 111 193 193 193 F F F 10 200-219 111 111 111 193 193 193 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 108 108 108 189 189 189 F F F 15 300-319 110 110 110 192 192 192 F F F 16 320-339 110 110 110 192 192 192 F F F 17 340-359 110 110 110 191 191 191 T T T 18 360-379 111 111 111 193 193 193 T T T 19 380-399 110 110 110 192 192 192 T T T 20 400-419 109 109 109 190 190 190 T T T 21 420-439 109 109 109 190 190 190 F F F 22 440-459 110 110 110 192 192 192 T T T 23 460-479 108 108 108 189 189 189 F F F 24 480-499 110 110 110 192 192 192 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 108 108 108 188 188 188 T T T 28 560-579 111 111 111 193 193 193 T T T 29 580-599 110 110 110 192 192 192 F F F Max gen: 111, Max flush: 193 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 112 112 112 194 194 194 F F F 1 020-039 108 108 108 189 189 189 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 F F F 5 100-119 112 112 112 194 194 194 F F F 6 120-139 110 110 110 192 192 192 F F F 7 140-159 112 112 112 194 194 194 F F F 8 160-179 110 110 110 192 192 192 F F F 9 180-199 111 111 111 193 193 193 F F F 10 200-219 111 111 111 193 193 193 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 108 108 108 189 189 189 F F F 15 300-319 110 110 110 192 192 192 F F F 16 320-339 110 110 110 192 192 192 F F F 17 340-359 112 112 112 194 194 194 F F F 18 360-379 112 112 112 194 194 194 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 112 112 112 194 194 194 F F F 21 420-439 109 109 109 190 190 190 F F F 22 440-459 112 112 112 194 194 194 F F F 23 460-479 108 108 108 189 189 189 F F F 24 480-499 110 110 110 192 192 192 F F F 25 500-519 106 106 106 184 184 184 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 112 112 112 194 194 194 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 112, Max flush: 194 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.578736786Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.624035042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.624703321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.624745745Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.624760702Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.62477554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} {"msg":"Crucible 883cf630-6b59-44fe-b018-58cd18d841de has session id: 98a85748-5775-4433-8eaf-eca49db53a7b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.625351902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Upstairs opts: Upstairs UUID: 883cf630-6b59-44fe-b018-58cd18d841de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.625395626Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Crucible stats registered with UUID: 883cf630-6b59-44fe-b018-58cd18d841de","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.625418633Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657041199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657073974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} {"msg":"883cf630-6b59-44fe-b018-58cd18d841de active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657150194Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657172821Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657201017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657218065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657234682Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657255379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657272157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657288155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657304793Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.65732245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62934)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657449323Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47026)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657490627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_connection connected from Ok(127.0.0.1:49651)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.657578765Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658151067Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658184943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658209869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658236396Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658264172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658289908Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658849712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658882687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.658915323Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659440821Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659470097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659485425Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659498183Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659511971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659526019Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659537967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659550976Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[0]R flush_numbers[0..12]: [194, 189, 189, 180, 183, 194, 192, 194, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659564674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[0]R generation[0..12]: [112, 108, 108, 102, 105, 112, 110, 112, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659578392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.65959221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[1]R flush_numbers[0..12]: [194, 189, 189, 180, 183, 194, 192, 194, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659620836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[1]R generation[0..12]: [112, 108, 108, 102, 105, 112, 110, 112, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659635524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659653062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[2]R flush_numbers[0..12]: [194, 189, 189, 180, 183, 194, 192, 194, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.65966704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[2]R generation[0..12]: [112, 108, 108, 102, 105, 112, 110, 112, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659680608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659694306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:09:28.659706954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659727332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Next flush: 195","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.6597401Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659752808Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"":"downstairs","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659765636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659777435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"0","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659790073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"1","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659802761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"client":"2","":"downstairs","session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659815489Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659827498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"883cf630-6b59-44fe-b018-58cd18d841de is now active with session: 98a85748-5775-4433-8eaf-eca49db53a7b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659840006Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659853734Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638,"session_id":"98a85748-5775-4433-8eaf-eca49db53a7b"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:28.659865883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 129 len 16384 data: 18 17 16 14 02/30 Flush 03/30 Write block 547 len 36864 data: 10 8 7 7 7 6 7 9 6 04/30 Read block 72 len 20480 05/30 Write block 14 len 32768 data: 13 14 14 12 13 13 12 10 06/30 Write block 466 len 8192 data: 5 6 07/30 Read block 349 len 20480 08/30 Read block 158 len 16384 09/30 Write block 318 len 4096 data: 13 10/30 Read block 559 len 20480 11/30 Flush 12/30 Write block 281 len 16384 data: 5 6 7 7 13/30 Write block 514 len 4096 data: 9 14/30 Read block 520 len 8192 15/30 Read block 38 len 4096 16/30 Write block 479 len 32768 data: 9 7 7 8 8 7 8 10 17/30 Flush 18/30 Write block 348 len 8192 data: 10 9 19/30 Write block 421 len 8192 data: 10 10 20/30 Read block 440 len 36864 21/30 Read block 98 len 32768 22/30 Read block 273 len 36864 23/30 Read block 82 len 36864 24/30 Write block 95 len 16384 data: 9 7 9 9 25/30 Write block 465 len 4096 data: 4 26/30 Write block 105 len 40960 data: 10 9 6 7 6 6 8 7 7 8 27/30 Write block 91 len 24576 data: 11 12 10 10 10 8 28/30 Read block 300 len 20480 29/30 Write block 334 len 40960 data: 8 9 12 10 12 16 18 15 12 14 30/30 Read block 95 len 28672 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 2 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Write 1 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:28.678767198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 105 105 105 183 183 183 T T T 5 100-119 112 112 112 194 194 194 T T T 6 120-139 114 114 114 195 195 195 F F F 7 140-159 112 112 112 194 194 194 F F F 8 160-179 110 110 110 192 192 192 F F F 9 180-199 111 111 111 193 193 193 F F F 10 200-219 111 111 111 193 193 193 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 114 114 114 196 196 196 F F F 16 320-339 110 110 110 192 192 192 T T T 17 340-359 112 112 112 194 194 194 T T T 18 360-379 112 112 112 194 194 194 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 112 112 112 194 194 194 F F F 21 420-439 109 109 109 190 190 190 T T T 22 440-459 112 112 112 194 194 194 F F F 23 460-479 114 114 114 197 197 197 T T T 24 480-499 114 114 114 197 197 197 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 114 114 114 196 196 196 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 114, Max flush: 197 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 115 115 115 198 198 198 F F F 5 100-119 115 115 115 198 198 198 F F F 6 120-139 114 114 114 195 195 195 F F F 7 140-159 112 112 112 194 194 194 F F F 8 160-179 110 110 110 192 192 192 F F F 9 180-199 111 111 111 193 193 193 F F F 10 200-219 111 111 111 193 193 193 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 114 114 114 196 196 196 F F F 16 320-339 115 115 115 198 198 198 F F F 17 340-359 115 115 115 198 198 198 F F F 18 360-379 112 112 112 194 194 194 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 112 112 112 194 194 194 F F F 21 420-439 115 115 115 198 198 198 F F F 22 440-459 112 112 112 194 194 194 F F F 23 460-479 115 115 115 198 198 198 F F F 24 480-499 114 114 114 197 197 197 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 114 114 114 196 196 196 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 115, Max flush: 198 Begin loop 58 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:32.960698421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.005891192Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.006531734Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.006571979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.006599815Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.006617843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} {"msg":"Crucible 59cd1019-0265-4b21-989a-5a84627c0a99 has session id: d465b84e-fd29-40d6-aa8a-eb3fba6c2305","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.007244837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Upstairs opts: Upstairs UUID: 59cd1019-0265-4b21-989a-5a84627c0a99, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.007321917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Crucible stats registered with UUID: 59cd1019-0265-4b21-989a-5a84627c0a99","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.007358851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038402047Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038428773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} {"msg":"59cd1019-0265-4b21-989a-5a84627c0a99 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038507413Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.03852493Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038549877Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038563445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038576583Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038589651Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038611888Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038641374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038667811Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038696037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_connection connected from Ok(127.0.0.1:40630)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038718704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_connection connected from Ok(127.0.0.1:41029)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038880541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_connection connected from Ok(127.0.0.1:52050)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.038927315Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.039347438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.039373994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.039577036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.039606502Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.039755392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.039784878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040159476Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040189283Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040482332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040751196Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040779312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.04079426Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040806638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040819986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.040833304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041051894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041080251Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[0]R flush_numbers[0..12]: [196, 196, 189, 180, 198, 198, 195, 194, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041095859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[0]R generation[0..12]: [114, 114, 108, 102, 115, 115, 114, 112, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041110526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041124464Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[1]R flush_numbers[0..12]: [196, 196, 189, 180, 198, 198, 195, 194, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041137983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[1]R generation[0..12]: [114, 114, 108, 102, 115, 115, 114, 112, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041151751Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041165549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[2]R flush_numbers[0..12]: [196, 196, 189, 180, 198, 198, 195, 194, 192, 193, 193, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041179527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[2]R generation[0..12]: [114, 114, 108, 102, 115, 115, 114, 112, 110, 111, 111, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041197035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041211843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Max found gen is 116","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-11T06:09:33.04123072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041243938Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Next flush: 199","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041256577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041269265Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"":"downstairs","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041282373Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041294191Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"0","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.04130717Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"1","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041320108Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"client":"2","":"downstairs","session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041333466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041345684Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"59cd1019-0265-4b21-989a-5a84627c0a99 is now active with session: d465b84e-fd29-40d6-aa8a-eb3fba6c2305","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041362242Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.04137431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646,"session_id":"d465b84e-fd29-40d6-aa8a-eb3fba6c2305"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:33.041386689Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 316 len 8192 data: 12 13 02/30 Flush 03/30 Write block 367 len 28672 data: 12 11 12 11 10 8 7 04/30 Write block 197 len 16384 data: 9 8 8 8 05/30 Write block 105 len 16384 data: 11 10 7 8 06/30 Write block 162 len 4096 data: 7 07/30 Read block 573 len 8192 08/30 Read block 391 len 12288 09/30 Write block 332 len 8192 data: 8 10 10/30 Read block 308 len 8192 11/30 Read block 431 len 8192 12/30 Read block 350 len 20480 13/30 Read block 340 len 40960 14/30 Read block 381 len 12288 15/30 Read block 488 len 4096 16/30 Write block 337 len 36864 data: 11 13 17 19 16 13 15 12 13 17/30 Write block 322 len 20480 data: 11 9 9 8 6 18/30 Flush 19/30 Write block 440 len 36864 data: 5 4 5 8 10 12 12 11 12 20/30 Read block 176 len 24576 21/30 Flush 22/30 Write block 98 len 12288 data: 10 8 7 23/30 Read block 226 len 36864 24/30 Read block 255 len 24576 25/30 Write block 156 len 32768 data: 12 11 9 8 9 9 8 7 26/30 Read block 30 len 4096 27/30 Write block 481 len 12288 data: 8 9 9 28/30 Read block 312 len 12288 29/30 Read block 417 len 16384 30/30 Write block 427 len 40960 data: 9 10 9 12 9 8 8 7 8 9 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:33.059906767Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 115 115 115 198 198 198 T T T 5 100-119 116 116 116 200 200 200 T T T 6 120-139 114 114 114 195 195 195 F F F 7 140-159 112 112 112 194 194 194 T T T 8 160-179 116 116 116 200 200 200 T T T 9 180-199 116 116 116 200 200 200 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 116 116 116 199 199 199 F F F 16 320-339 116 116 116 200 200 200 F F F 17 340-359 116 116 116 200 200 200 F F F 18 360-379 116 116 116 200 200 200 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 112 112 112 194 194 194 F F F 21 420-439 115 115 115 198 198 198 T T T 22 440-459 116 116 116 201 201 201 F F F 23 460-479 115 115 115 198 198 198 F F F 24 480-499 114 114 114 197 197 197 T T T 25 500-519 114 114 114 197 197 197 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 114 114 114 196 196 196 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 116, Max flush: 201 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 117 117 117 202 202 202 F F F 5 100-119 117 117 117 202 202 202 F F F 6 120-139 114 114 114 195 195 195 F F F 7 140-159 117 117 117 202 202 202 F F F 8 160-179 117 117 117 202 202 202 F F F 9 180-199 116 116 116 200 200 200 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 116 116 116 199 199 199 F F F 16 320-339 116 116 116 200 200 200 F F F 17 340-359 116 116 116 200 200 200 F F F 18 360-379 116 116 116 200 200 200 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 112 112 112 194 194 194 F F F 21 420-439 117 117 117 202 202 202 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 115 115 115 198 198 198 F F F 24 480-499 117 117 117 202 202 202 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 114 114 114 196 196 196 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 117, Max flush: 202 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.345955607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:37.390425187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.390785927Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.422650181Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423212434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423246949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423264167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423278535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} {"msg":"Crucible 1fe58295-92fd-4f48-af48-fac52e692c28 has session id: e1501a44-a5de-4f39-a0d1-35ac61fc0b00","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423864235Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Upstairs opts: Upstairs UUID: 1fe58295-92fd-4f48-af48-fac52e692c28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423925326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Crucible stats registered with UUID: 1fe58295-92fd-4f48-af48-fac52e692c28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.423952333Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.454988329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455031353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} {"msg":"1fe58295-92fd-4f48-af48-fac52e692c28 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455080327Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455098914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45512636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455146367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455163715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455180163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455196851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455261872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455452836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45556676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_connection connected from Ok(127.0.0.1:43973)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455610654Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_connection connected from Ok(127.0.0.1:33484)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45571555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_connection connected from Ok(127.0.0.1:38177)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.455733617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456109416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456147601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456176077Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456204683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456228359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456247507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456774495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45681395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.456845545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457357525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457385121Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457402589Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457471629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457504045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457521533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457533831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457547149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[0]R flush_numbers[0..12]: [196, 196, 189, 180, 202, 202, 195, 202, 202, 200, 200, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457560737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[0]R generation[0..12]: [114, 114, 108, 102, 117, 117, 114, 117, 117, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457575235Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457589424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[1]R flush_numbers[0..12]: [196, 196, 189, 180, 202, 202, 195, 202, 202, 200, 200, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457602912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[1]R generation[0..12]: [114, 114, 108, 102, 117, 117, 114, 117, 117, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45761634Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457629988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[2]R flush_numbers[0..12]: [196, 196, 189, 180, 202, 202, 195, 202, 202, 200, 200, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457643256Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[2]R generation[0..12]: [114, 114, 108, 102, 117, 117, 114, 117, 117, 116, 116, 111]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:09:37.457656714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45768573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457710317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457733644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.45775711Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457774098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"":"downstairs","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457788306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457800005Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"0","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457813153Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"1","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457826921Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"client":"2","":"downstairs","session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457843599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457855697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"1fe58295-92fd-4f48-af48-fac52e692c28 is now active with session: e1501a44-a5de-4f39-a0d1-35ac61fc0b00","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457868205Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457880714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654,"session_id":"e1501a44-a5de-4f39-a0d1-35ac61fc0b00"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:37.457893632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 142 len 4096 data: 9 02/30 Read block 311 len 28672 03/30 Write block 371 len 32768 data: 11 9 8 10 12 11 13 15 04/30 Read block 426 len 20480 05/30 Read block 209 len 32768 06/30 Read block 446 len 40960 07/30 Read block 492 len 8192 08/30 Read block 135 len 24576 09/30 Write block 111 len 32768 data: 9 8 8 9 8 7 7 6 10/30 Flush 11/30 Write block 129 len 32768 data: 19 18 17 15 12 11 12 10 12/30 Read block 189 len 16384 13/30 Read block 527 len 12288 14/30 Write block 427 len 16384 data: 10 11 10 13 15/30 Read block 434 len 36864 16/30 Read block 89 len 40960 17/30 Write block 360 len 8192 data: 10 10 18/30 Read block 550 len 16384 19/30 Flush 20/30 Write block 320 len 28672 data: 12 11 12 10 10 9 7 21/30 Read block 349 len 36864 22/30 Read block 560 len 20480 23/30 Read block 477 len 36864 24/30 Write block 356 len 24576 data: 9 8 10 9 11 11 25/30 Flush 26/30 Write block 463 len 24576 data: 9 6 5 6 7 6 27/30 Write block 271 len 16384 data: 16 14 12 12 28/30 Write block 96 len 4096 data: 9 29/30 Read block 30 len 12288 30/30 Write block 405 len 32768 data: 7 8 10 10 10 9 9 11 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:37.478153412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 117 117 117 202 202 202 T T T 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 118 118 118 203 203 203 F F F 8 160-179 117 117 117 202 202 202 F F F 9 180-199 116 116 116 200 200 200 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 108 108 108 188 188 188 T T T 14 280-299 114 114 114 197 197 197 F F F 15 300-319 116 116 116 199 199 199 F F F 16 320-339 118 118 118 205 205 205 F F F 17 340-359 118 118 118 205 205 205 F F F 18 360-379 118 118 118 205 205 205 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 112 112 112 194 194 194 T T T 21 420-439 118 118 118 204 204 204 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 115 115 115 198 198 198 T T T 24 480-499 117 117 117 202 202 202 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 114 114 114 196 196 196 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 118, Max flush: 205 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 118 118 118 203 203 203 F F F 8 160-179 117 117 117 202 202 202 F F F 9 180-199 116 116 116 200 200 200 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 111 111 111 193 193 193 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 116 116 116 199 199 199 F F F 16 320-339 118 118 118 205 205 205 F F F 17 340-359 118 118 118 205 205 205 F F F 18 360-379 118 118 118 205 205 205 F F F 19 380-399 112 112 112 194 194 194 F F F 20 400-419 119 119 119 206 206 206 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 119 119 119 206 206 206 F F F 24 480-499 117 117 117 202 202 202 F F F 25 500-519 114 114 114 197 197 197 F F F 26 520-539 108 108 108 188 188 188 F F F 27 540-559 114 114 114 196 196 196 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 110 110 110 192 192 192 F F F Max gen: 119, Max flush: 206 Begin loop 60 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.783343254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.827819403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.828478013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.828533155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.828554922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.828577249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} {"msg":"Crucible eb393b5a-3c7f-4d9c-81a3-83e7a57b64b4 has session id: 0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.829196345Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Upstairs opts: Upstairs UUID: eb393b5a-3c7f-4d9c-81a3-83e7a57b64b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.829240728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Crucible stats registered with UUID: eb393b5a-3c7f-4d9c-81a3-83e7a57b64b4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.829266105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860396778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860433453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} {"msg":"eb393b5a-3c7f-4d9c-81a3-83e7a57b64b4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860474958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860502704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860546418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860581603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860615858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860647214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.86067967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860717724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.86075224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860786435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:63209)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.86082215Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:36177)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860854856Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_connection connected from Ok(127.0.0.1:57333)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.860891161Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.861523844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.861549521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.861727736Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.861766611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.861796777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.861822153Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.862079518Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.862423281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.862463696Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.862743878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.862781382Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.86279807Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.862991134Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863027849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863052115Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863076932Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863101189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[0]R flush_numbers[0..12]: [196, 196, 189, 180, 206, 203, 204, 203, 202, 200, 200, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863123795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[0]R generation[0..12]: [114, 114, 108, 102, 119, 118, 118, 118, 117, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863145672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863169909Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[1]R flush_numbers[0..12]: [196, 196, 189, 180, 206, 203, 204, 203, 202, 200, 200, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863195686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[1]R generation[0..12]: [114, 114, 108, 102, 119, 118, 118, 118, 117, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863221782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863246199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[2]R flush_numbers[0..12]: [196, 196, 189, 180, 206, 203, 204, 203, 202, 200, 200, 193]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863272495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R generation[0..12]: [114, 114, 108, 102, 119, 118, 118, 118, 117, 116, 116, 111]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863304971Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863333957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863360133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.86338687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863416346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863443562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"":"downstairs","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863471098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863495725Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"0","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"}Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863526291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"1","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863547148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"client":"2","":"downstairs","session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863560806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863573194Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"eb393b5a-3c7f-4d9c-81a3-83e7a57b64b4 is now active with session: 0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863585792Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863597631Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662,"session_id":"0cc23bdb-6d06-4c73-b8f4-9a19fdfadeb1"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:41.863609699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 493 len 40960 data: 13 12 14 12 13 12 11 10 9 6 02/30 Read block 291 len 16384 03/30 Write block 310 len 20480 data: 6 6 5 7 10 04/30 Read block 172 len 16384 05/30 Read block 114 len 16384 06/30 Read block 280 len 8192 07/30 Write block 222 len 8192 data: 12 11 08/30 Read block 153 len 40960 09/30 Write block 505 len 8192 data: 8 8 10/30 Write block 590 len 28672 data: 8 9 8 8 7 5 4 11/30 Read block 384 len 16384 12/30 Read block 274 len 8192 13/30 Flush 14/30 Write block 407 len 40960 data: 11 11 11 10 10 12 10 10 9 9 15/30 Write block 158 len 12288 data: 10 9 10 16/30 Read block 423 len 16384 17/30 Read block 555 len 40960 18/30 Read block 281 len 12288 19/30 Write block 397 len 4096 data: 11 20/30 Read block 80 len 20480 21/30 Write block 514 len 12288 data: 10 8 7 22/30 Read block 294 len 28672 23/30 Write block 534 len 40960 data: 8 8 6 6 5 8 9 8 8 8 24/30 Read block 50 len 8192 25/30 Read block 329 len 40960 26/30 Write block 475 len 20480 data: 10 10 10 10 10 27/30 Write block 589 len 20480 data: 8 9 10 9 9 28/30 Write block 412 len 32768 data: 13 11 11 10 10 10 10 9 29/30 Read block 253 len 8192 30/30 Write block 355 len 32768 data: 10 10 9 11 10 12 12 10 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 10 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 3 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:41.883698312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 118 118 118 203 203 203 T T T 8 160-179 117 117 117 202 202 202 T T T 9 180-199 116 116 116 200 200 200 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 120 120 120 207 207 207 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 120 120 120 207 207 207 F F F 16 320-339 118 118 118 205 205 205 F F F 17 340-359 118 118 118 205 205 205 T T T 18 360-379 118 118 118 205 205 205 T T T 19 380-399 112 112 112 194 194 194 T T T 20 400-419 119 119 119 206 206 206 T T T 21 420-439 118 118 118 204 204 204 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 119 119 119 206 206 206 T T T 24 480-499 120 120 120 207 207 207 F F F 25 500-519 120 120 120 207 207 207 T T T 26 520-539 108 108 108 188 188 188 T T T 27 540-559 114 114 114 196 196 196 T T T 28 560-579 112 112 112 194 194 194 F F F 29 580-599 120 120 120 207 207 207 T T T Max gen: 120, Max flush: 207 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 114 114 114 196 196 196 F F F 2 040-059 108 108 108 189 189 189 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 121 121 121 208 208 208 F F F 8 160-179 121 121 121 208 208 208 F F F 9 180-199 116 116 116 200 200 200 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 120 120 120 207 207 207 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 120 120 120 207 207 207 F F F 16 320-339 118 118 118 205 205 205 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 121 121 121 208 208 208 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 121 121 121 208 208 208 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 121 121 121 208 208 208 F F F 24 480-499 120 120 120 207 207 207 F F F 25 500-519 121 121 121 208 208 208 F F F 26 520-539 121 121 121 208 208 208 F F F 27 540-559 121 121 121 208 208 208 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 121 121 121 208 208 208 F F F Max gen: 121, Max flush: 208 Begin loop 61 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.193401568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:46.236557937Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.236956972Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.268855831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.269482685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.26951774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.269538988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.269553395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} {"msg":"Crucible e5379f59-808a-4829-afc5-54f857e2da62 has session id: 7998fb06-4ef9-4fe2-a294-264352ede28d","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.270049798Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Upstairs opts: Upstairs UUID: e5379f59-808a-4829-afc5-54f857e2da62, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.270111719Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Crucible stats registered with UUID: e5379f59-808a-4829-afc5-54f857e2da62","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.270135946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301467032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301497298Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} {"msg":"e5379f59-808a-4829-afc5-54f857e2da62 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301569588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301587055Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301611292Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.30162504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301639498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301652586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301665525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301678483Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301691961Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301707489Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_connection connected from Ok(127.0.0.1:55891)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301802836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34142)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301840001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33413)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.301872756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.302573931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.302601217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.302616665Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.302630733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.302643861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.302679556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303227281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303362883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303390069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.3036774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303705836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303721144Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303933655Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303956582Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303976299Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.303990557Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304010314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[0]R flush_numbers[0..12]: [196, 196, 189, 180, 206, 203, 204, 208, 208, 200, 200, 207]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304031241Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[0]R generation[0..12]: [114, 114, 108, 102, 119, 118, 118, 121, 121, 116, 116, 120]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304046979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304060807Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[1]R flush_numbers[0..12]: [196, 196, 189, 180, 206, 203, 204, 208, 208, 200, 200, 207]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304074335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[1]R generation[0..12]: [114, 114, 108, 102, 119, 118, 118, 121, 121, 116, 116, 120]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304087804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304101512Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[2]R flush_numbers[0..12]: [196, 196, 189, 180, 206, 203, 204, 208, 208, 200, 200, 207]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.30411494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[2]R generation[0..12]: [114, 114, 108, 102, 119, 118, 118, 121, 121, 116, 116, 120]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304128488Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304142126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Max found gen is 122","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2025-01-11T06:09:46.304161254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304180791Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Next flush: 209","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304197789Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304210847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"":"downstairs","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304224045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304235883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"0","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"timeDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"2025-01-11T06:09:46.304248951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"1","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304265319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"client":"2","":"downstairs","session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304278178Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304291006Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"e5379f59-808a-4829-afc5-54f857e2da62 is now active with session: 7998fb06-4ef9-4fe2-a294-264352ede28d","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304303524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304315453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670,"session_id":"7998fb06-4ef9-4fe2-a294-264352ede28d"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:46.304327421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 228 len 16384 data: 7 7 8 8 02/30 Flush 03/30 Write block 471 len 36864 data: 7 7 8 11 11 11 11 11 11 04/30 Read block 324 len 20480 05/30 Read block 529 len 4096 06/30 Write block 516 len 12288 data: 8 7 6 07/30 Write block 378 len 4096 data: 16 08/30 Read block 199 len 20480 09/30 Write block 176 len 24576 data: 7 9 9 8 8 7 10/30 Read block 389 len 24576 11/30 Read block 543 len 24576 12/30 Write block 32 len 12288 data: 7 8 10 13/30 Read block 245 len 8192 14/30 Write block 408 len 40960 data: 12 12 11 11 14 12 12 11 11 11 15/30 Read block 459 len 20480 16/30 Read block 551 len 8192 17/30 Write block 165 len 16384 data: 8 9 10 11 18/30 Write block 485 len 40960 data: 9 11 10 12 9 10 11 12 14 13 19/30 Read block 397 len 32768 20/30 Read block 508 len 32768 21/30 Read block 572 len 16384 22/30 Write block 324 len 16384 data: 11 10 8 6 23/30 Read block 140 len 8192 24/30 Read block 268 len 24576 25/30 Read block 162 len 20480 26/30 Flush 27/30 Write block 36 len 36864 data: 10 10 9 9 10 13 15 14 15 28/30 Write block 162 len 8192 data: 9 8 29/30 Write block 513 len 16384 data: 11 11 9 9 30/30 Read block 195 len 12288 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 9 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 4 Done Sent Done false 1029 Acked Read 3 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 0 3 Done 3 2 4 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:46.323517637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 122 122 122 210 210 210 T T T 2 040-059 108 108 108 189 189 189 T T T 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 121 121 121 208 208 208 F F F 8 160-179 122 122 122 210 210 210 T T T 9 180-199 122 122 122 210 210 210 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 122 122 122 209 209 209 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 120 120 120 207 207 207 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 122 122 122 210 210 210 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 122 122 122 210 210 210 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 122 122 122 210 210 210 F F F 24 480-499 122 122 122 210 210 210 F F F 25 500-519 122 122 122 210 210 210 T T T 26 520-539 121 121 121 208 208 208 F F F 27 540-559 121 121 121 208 208 208 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 121 121 121 208 208 208 F F F Max gen: 122, Max flush: 210 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 123 123 123 211 211 211 F F F 2 040-059 123 123 123 211 211 211 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 121 121 121 208 208 208 F F F 8 160-179 123 123 123 211 211 211 F F F 9 180-199 122 122 122 210 210 210 F F F 10 200-219 116 116 116 200 200 200 F F F 11 220-239 122 122 122 209 209 209 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 114 114 114 197 197 197 F F F 15 300-319 120 120 120 207 207 207 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 122 122 122 210 210 210 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 122 122 122 210 210 210 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 116 116 116 201 201 201 F F F 23 460-479 122 122 122 210 210 210 F F F 24 480-499 122 122 122 210 210 210 F F F 25 500-519 123 123 123 211 211 211 F F F 26 520-539 121 121 121 208 208 208 F F F 27 540-559 121 121 121 208 208 208 F F F 28 560-579 112 112 112 194 194 194 F F F 29 580-599 121 121 121 208 208 208 F F F Max gen: 123, Max flush: 211 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.626280172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.673412187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674159375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674207599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674225686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674262671Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} {"msg":"Crucible 6fb801dd-b131-489b-904a-bbbfe0ef87b1 has session id: 58fd1e2a-88f1-4872-b18f-8fe955cf6789","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674821865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Upstairs opts: Upstairs UUID: 6fb801dd-b131-489b-904a-bbbfe0ef87b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674861129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Crucible stats registered with UUID: 6fb801dd-b131-489b-904a-bbbfe0ef87b1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.674880127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707511425Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707553139Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} {"msg":"6fb801dd-b131-489b-904a-bbbfe0ef87b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707630749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707662924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.70769486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707716557Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707733705Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707755431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707784398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707813943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.70784255Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707873415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_connection connected from Ok(127.0.0.1:43496)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707905171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_connection connected from Ok(127.0.0.1:35158)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.707937067Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_connection connected from Ok(127.0.0.1:65394)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.708017336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.708714551Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.708741027Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.708767803Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.70879247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.708818726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.708844393Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709352103Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709436892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709467017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709854964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709884611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709899149Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709978998Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.709997245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710010323Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710068285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710086773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[0]R flush_numbers[0..12]: [196, 211, 211, 180, 206, 203, 204, 208, 211, 210, 200, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710101251Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[0]R generation[0..12]: [114, 123, 123, 102, 119, 118, 118, 121, 123, 122, 116, 122]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710118028Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710132107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[1]R flush_numbers[0..12]: [196, 211, 211, 180, 206, 203, 204, 208, 211, 210, 200, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710145445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[1]R generation[0..12]: [114, 123, 123, 102, 119, 118, 118, 121, 123, 122, 116, 122]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710158823Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710172401Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[2]R flush_numbers[0..12]: [196, 211, 211, 180, 206, 203, 204, 208, 211, 210, 200, 209]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710185869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[2]R generation[0..12]: [114, 123, 123, 102, 119, 118, 118, 121, 123, 122, 116, 122]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710199377Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710213035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710225514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710238342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.71025091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2025-01-11T06:09:50.710263369Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"":"downstairs","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710283676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710295384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"0","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710309872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"1","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.7103229Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"client":"2","":"downstairs","session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710335769Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710347617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"6fb801dd-b131-489b-904a-bbbfe0ef87b1 is now active with session: 58fd1e2a-88f1-4872-b18f-8fe955cf6789","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710360066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710371664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678,"session_id":"58fd1e2a-88f1-4872-b18f-8fe955cf6789"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:50.710383532Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 295 len 36864 data: 11 12 13 15 15 13 11 10 8 02/30 Read block 75 len 16384 03/30 Flush 04/30 Write block 315 len 4096 data: 9 05/30 Write block 469 len 20480 data: 7 7 8 8 9 06/30 Write block 314 len 8192 data: 11 10 07/30 Write block 578 len 24576 data: 8 7 8 9 13 10 08/30 Write block 33 len 4096 data: 9 09/30 Write block 217 len 4096 data: 11 10/30 Read block 387 len 32768 11/30 Read block 198 len 32768 12/30 Read block 59 len 16384 13/30 Read block 91 len 40960 14/30 Read block 374 len 8192 15/30 Write block 226 len 4096 data: 9 16/30 Write block 454 len 16384 data: 9 8 8 9 17/30 Write block 464 len 8192 data: 7 6 18/30 Write block 526 len 16384 data: 9 11 11 10 19/30 Write block 217 len 20480 data: 12 10 10 10 9 20/30 Read block 188 len 36864 21/30 Write block 167 len 28672 data: 11 12 11 10 10 12 13 22/30 Write block 363 len 36864 data: 10 9 9 10 13 12 13 12 12 23/30 Write block 441 len 36864 data: 5 6 9 11 13 13 12 13 14 24/30 Write block 225 len 20480 data: 11 10 6 8 8 25/30 Flush 26/30 Write block 544 len 8192 data: 11 11 27/30 Write block 229 len 36864 data: 9 9 9 8 9 11 12 12 14 28/30 Write block 550 len 20480 data: 8 8 7 8 10 29/30 Write block 475 len 28672 data: 12 12 12 12 12 8 9 30/30 Read block 3 len 4096 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 4 5 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:50.727786203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 124 124 124 213 213 213 F F F 2 040-059 123 123 123 211 211 211 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 121 121 121 208 208 208 F F F 8 160-179 124 124 124 213 213 213 F F F 9 180-199 122 122 122 210 210 210 F F F 10 200-219 124 124 124 213 213 213 F F F 11 220-239 124 124 124 213 213 213 T T T 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 124 124 124 212 212 212 F F F 15 300-319 124 124 124 213 213 213 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 124 124 124 213 213 213 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 122 122 122 210 210 210 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 124 124 124 213 213 213 F F F 23 460-479 124 124 124 213 213 213 T T T 24 480-499 122 122 122 210 210 210 T T T 25 500-519 123 123 123 211 211 211 F F F 26 520-539 124 124 124 213 213 213 F F F 27 540-559 121 121 121 208 208 208 T T T 28 560-579 124 124 124 213 213 213 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 124, Max flush: 213 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 124 124 124 213 213 213 F F F 2 040-059 123 123 123 211 211 211 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 121 121 121 208 208 208 F F F 8 160-179 124 124 124 213 213 213 F F F 9 180-199 122 122 122 210 210 210 F F F 10 200-219 124 124 124 213 213 213 F F F 11 220-239 125 125 125 214 214 214 F F F 12 240-259 111 111 111 193 193 193 F F F 13 260-279 119 119 119 206 206 206 F F F 14 280-299 124 124 124 212 212 212 F F F 15 300-319 124 124 124 213 213 213 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 124 124 124 213 213 213 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 122 122 122 210 210 210 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 124 124 124 213 213 213 F F F 23 460-479 125 125 125 214 214 214 F F F 24 480-499 125 125 125 214 214 214 F F F 25 500-519 123 123 123 211 211 211 F F F 26 520-539 124 124 124 213 213 213 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 124 124 124 213 213 213 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 125, Max flush: 214 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.001076248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.045948522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.046603663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.046653756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.046672563Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.046690691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} {"msg":"Crucible 4128803b-7a1f-4c51-8a4e-bec917451208 has session id: 23d1bf5c-6685-49d2-ac25-6e972d4e04cf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.047255284Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Upstairs opts: Upstairs UUID: 4128803b-7a1f-4c51-8a4e-bec917451208, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.047305257Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Crucible stats registered with UUID: 4128803b-7a1f-4c51-8a4e-bec917451208","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.047333973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078403905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.07843681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} {"msg":"4128803b-7a1f-4c51-8a4e-bec917451208 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078468766Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078486704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.07851288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078529967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078552784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078570502Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.07858686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078603237Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078620165Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078636543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:45254)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.07866188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:63872)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078866072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_connection connected from Ok(127.0.0.1:33757)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.078984096Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.079553408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.079581034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.079712636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.079737353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.079800824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.079842338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080147586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080188171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.0804107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.08070797Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080745595Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080772711Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080834183Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080862069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080876947Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080889095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080903373Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[0]R flush_numbers[0..12]: [196, 213, 211, 180, 206, 203, 204, 208, 213, 210, 213, 214]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080919421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[0]R generation[0..12]: [114, 124, 123, 102, 119, 118, 118, 121, 124, 122, 124, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080933309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080947097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[1]R flush_numbers[0..12]: [196, 213, 211, 180, 206, 203, 204, 208, 213, 210, 213, 214]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080960585Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[1]R generation[0..12]: [114, 124, 123, 102, 119, 118, 118, 121, 124, 122, 124, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080977763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.080991841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[2]R flush_numbers[0..12]: [196, 213, 211, 180, 206, 203, 204, 208, 213, 210, 213, 214]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081005389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[2]R generation[0..12]: [114, 124, 123, 102, 119, 118, 118, 121, 124, 122, 124, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081019797Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081033555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081046263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081059182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.08107196Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081084778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"":"downstairs","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:09:55.081098047Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081114604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"0","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081127672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"1","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081140441Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"client":"2","":"downstairs","session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081153349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081165297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"4128803b-7a1f-4c51-8a4e-bec917451208 is now active with session: 23d1bf5c-6685-49d2-ac25-6e972d4e04cf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081180485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081192554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686,"session_id":"23d1bf5c-6685-49d2-ac25-6e972d4e04cf"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:55.081204482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 246 len 28672 data: 11 10 10 10 12 12 11 02/30 Read block 233 len 24576 03/30 Write block 517 len 4096 data: 8 04/30 Read block 160 len 4096 05/30 Read block 404 len 36864 06/30 Write block 41 len 40960 data: 14 16 15 16 16 14 12 13 11 8 07/30 Read block 249 len 36864 08/30 Read block 339 len 36864 09/30 Write block 109 len 20480 data: 7 7 10 9 9 10/30 Read block 340 len 12288 11/30 Read block 300 len 32768 12/30 Read block 65 len 32768 13/30 Write block 274 len 16384 data: 13 10 13 13 14/30 Read block 424 len 28672 15/30 Write block 143 len 32768 data: 11 13 14 12 12 12 11 11 16/30 Write block 302 len 24576 data: 11 9 9 6 7 6 17/30 Read block 450 len 16384 18/30 Write block 573 len 12288 data: 7 7 6 19/30 Write block 175 len 28672 data: 9 8 10 10 9 9 8 20/30 Write block 205 len 40960 data: 8 10 11 12 11 11 10 10 11 9 21/30 Write block 167 len 40960 data: 12 13 12 11 11 13 14 11 10 9 22/30 Read block 543 len 32768 23/30 Write block 173 len 20480 data: 15 12 11 10 11 24/30 Read block 462 len 12288 25/30 Read block 228 len 36864 26/30 Read block 237 len 4096 27/30 Write block 464 len 20480 data: 8 7 7 8 7 28/30 Write block 172 len 20480 data: 14 16 13 12 11 29/30 Read block 265 len 8192 30/30 Read block 194 len 20480 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Write 10 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 30 30 29 89 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:55.102103674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 124 124 124 213 213 213 F F F 2 040-059 123 123 123 211 211 211 T T T 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 118 118 118 203 203 203 T T T 6 120-139 118 118 118 204 204 204 F F F 7 140-159 121 121 121 208 208 208 T T T 8 160-179 124 124 124 213 213 213 T T T 9 180-199 122 122 122 210 210 210 T T T 10 200-219 124 124 124 213 213 213 T T T 11 220-239 125 125 125 214 214 214 F F F 12 240-259 111 111 111 193 193 193 T T T 13 260-279 119 119 119 206 206 206 T T T 14 280-299 124 124 124 212 212 212 F F F 15 300-319 124 124 124 213 213 213 T T T 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 124 124 124 213 213 213 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 122 122 122 210 210 210 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 124 124 124 213 213 213 F F F 23 460-479 125 125 125 214 214 214 T T T 24 480-499 125 125 125 214 214 214 F F F 25 500-519 123 123 123 211 211 211 T T T 26 520-539 124 124 124 213 213 213 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 124 124 124 213 213 213 T T T 29 580-599 124 124 124 213 213 213 F F F Max gen: 125, Max flush: 214 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 124 124 124 213 213 213 F F F 2 040-059 126 126 126 215 215 215 F F F 3 060-079 102 102 102 180 180 180 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 126 126 126 215 215 215 F F F 8 160-179 126 126 126 215 215 215 F F F 9 180-199 126 126 126 215 215 215 F F F 10 200-219 126 126 126 215 215 215 F F F 11 220-239 125 125 125 214 214 214 F F F 12 240-259 126 126 126 215 215 215 F F F 13 260-279 126 126 126 215 215 215 F F F 14 280-299 124 124 124 212 212 212 F F F 15 300-319 126 126 126 215 215 215 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 124 124 124 213 213 213 F F F 19 380-399 121 121 121 208 208 208 F F F 20 400-419 122 122 122 210 210 210 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 124 124 124 213 213 213 F F F 23 460-479 126 126 126 215 215 215 F F F 24 480-499 125 125 125 214 214 214 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 124 124 124 213 213 213 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 126 126 126 215 215 215 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 126, Max flush: 215 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.412898001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.459227016Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.459944238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.459987382Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.4600045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.460022368Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} {"msg":"Crucible fb21e0ba-11d3-4cf8-b286-1907cc975462 has session id: e98830d9-e613-4b1c-9d1a-6d2fd0dae64f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.460692396Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Upstairs opts: Upstairs UUID: fb21e0ba-11d3-4cf8-b286-1907cc975462, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.460747678Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Crucible stats registered with UUID: fb21e0ba-11d3-4cf8-b286-1907cc975462","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.460775434Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492773899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492812574Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} {"msg":"fb21e0ba-11d3-4cf8-b286-1907cc975462 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49284367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492863537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492889414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492910751Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492935717Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.492960684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49298704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493013277Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493040863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493069799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55828)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493098745Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_connection connected from Ok(127.0.0.1:50685)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493127631Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56078)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493155967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49393917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493967906Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.493982924Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49401495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.494054174Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.494071472Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.494655872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.494699076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.494734511Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495290205Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495317332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.4953316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495344098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495357316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495370184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495382053Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495395041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[0]R flush_numbers[0..12]: [196, 213, 215, 180, 206, 215, 204, 215, 215, 215, 215, 214]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495408559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[0]R generation[0..12]: [114, 124, 126, 102, 119, 126, 118, 126, 126, 126, 126, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495422077Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495440705Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[1]R flush_numbers[0..12]: [196, 213, 215, 180, 206, 215, 204, 215, 215, 215, 215, 214]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495454803Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[1]R generation[0..12]: [114, 124, 126, 102, 119, 126, 118, 126, 126, 126, 126, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49547773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495504606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[2]R flush_numbers[0..12]: [196, 213, 215, 180, 206, 215, 204, 215, 215, 215, 215, 214]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495530613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[2]R generation[0..12]: [114, 124, 126, 102, 119, 126, 118, 126, 126, 126, 126, 125]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49554597Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495559949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495575416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:09:59.495598843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Next flush: 216","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49562428Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495649806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"":"downstairs","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495666594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495678602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"0","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49569287Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"1","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495715957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"client":"2","":"downstairs","session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495744983Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.49576856Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"fb21e0ba-11d3-4cf8-b286-1907cc975462 is now active with session: e98830d9-e613-4b1c-9d1a-6d2fd0dae64f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495782988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495801516Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694,"session_id":"e98830d9-e613-4b1c-9d1a-6d2fd0dae64f"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:09:59.495824142Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 194 len 40960 data: 6 7 6 10 9 9 9 8 7 7 02/30 Read block 284 len 40960 03/30 Read block 588 len 4096 04/30 Read block 146 len 12288 05/30 Write block 247 len 40960 data: 11 11 11 13 13 12 11 11 11 13 06/30 Write block 407 len 24576 data: 12 13 13 12 12 15 07/30 Read block 1 len 16384 08/30 Read block 74 len 32768 09/30 Read block 447 len 36864 10/30 Write block 525 len 40960 data: 10 10 12 12 11 10 8 8 7 9 11/30 Read block 234 len 16384 12/30 Write block 234 len 8192 data: 12 13 13/30 Write block 276 len 16384 data: 14 14 12 8 14/30 Read block 219 len 12288 15/30 Read block 238 len 16384 16/30 Write block 60 len 40960 data: 15 14 11 11 10 7 8 9 6 6 17/30 Read block 233 len 20480 18/30 Write block 533 len 12288 data: 8 10 9 19/30 Read block 171 len 40960 20/30 Read block 208 len 28672 21/30 Read block 295 len 40960 22/30 Read block 545 len 28672 23/30 Write block 384 len 32768 data: 12 14 17 15 17 12 11 12 24/30 Write block 567 len 8192 data: 8 8 25/30 Flush 26/30 Write block 389 len 16384 data: 13 12 13 11 27/30 Read block 263 len 36864 28/30 Read block 561 len 8192 29/30 Read block 574 len 40960 30/30 Write block 216 len 20480 data: 12 13 11 11 11 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:09:59.518596999Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 124 124 124 213 213 213 F F F 2 040-059 126 126 126 215 215 215 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 126 126 126 215 215 215 F F F 8 160-179 126 126 126 215 215 215 F F F 9 180-199 128 128 128 216 216 216 F F F 10 200-219 128 128 128 216 216 216 T T T 11 220-239 128 128 128 216 216 216 T T T 12 240-259 128 128 128 216 216 216 F F F 13 260-279 128 128 128 216 216 216 F F F 14 280-299 124 124 124 212 212 212 F F F 15 300-319 126 126 126 215 215 215 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 124 124 124 213 213 213 F F F 19 380-399 128 128 128 216 216 216 T T T 20 400-419 128 128 128 216 216 216 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 124 124 124 213 213 213 F F F 23 460-479 126 126 126 215 215 215 F F F 24 480-499 125 125 125 214 214 214 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 128 128 128 216 216 216 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 128, Max flush: 216 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 196 196 196 F F F 1 020-039 124 124 124 213 213 213 F F F 2 040-059 126 126 126 215 215 215 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 118 118 118 204 204 204 F F F 7 140-159 126 126 126 215 215 215 F F F 8 160-179 126 126 126 215 215 215 F F F 9 180-199 128 128 128 216 216 216 F F F 10 200-219 129 129 129 217 217 217 F F F 11 220-239 129 129 129 217 217 217 F F F 12 240-259 128 128 128 216 216 216 F F F 13 260-279 128 128 128 216 216 216 F F F 14 280-299 124 124 124 212 212 212 F F F 15 300-319 126 126 126 215 215 215 F F F 16 320-339 122 122 122 210 210 210 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 124 124 124 213 213 213 F F F 19 380-399 129 129 129 217 217 217 F F F 20 400-419 128 128 128 216 216 216 F F F 21 420-439 118 118 118 204 204 204 F F F 22 440-459 124 124 124 213 213 213 F F F 23 460-479 126 126 126 215 215 215 F F F 24 480-499 125 125 125 214 214 214 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 128 128 128 216 216 216 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 129, Max flush: 217 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.786974424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.83183574Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.832533605Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.832584348Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.832603125Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.832621453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} {"msg":"Crucible 5856c2d2-ef69-4a56-a6c7-6a22c5f325a4 has session id: 9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.83322495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Upstairs opts: Upstairs UUID: 5856c2d2-ef69-4a56-a6c7-6a22c5f325a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.833268984Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Crucible stats registered with UUID: 5856c2d2-ef69-4a56-a6c7-6a22c5f325a4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.833292801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864575333Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864608649Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} {"msg":"5856c2d2-ef69-4a56-a6c7-6a22c5f325a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864659202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864679279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864706686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864724083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864741031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864763188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864780005Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864795873Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864812551Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864840407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63390)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.864954232Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:34983)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865002645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_connection connected from Ok(127.0.0.1:63942)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865034411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865693931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865721607Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.86584294Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865867497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865887224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.865902652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866338532Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866367779Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866637622Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866896366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866916344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866930091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.866999222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867026468Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867041276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867171349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867189976Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[0]R flush_numbers[0..12]: [196, 213, 215, 216, 206, 215, 204, 215, 215, 216, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867204494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[0]R generation[0..12]: [114, 124, 126, 128, 119, 126, 118, 126, 126, 128, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867218292Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.86723199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[1]R flush_numbers[0..12]: [196, 213, 215, 216, 206, 215, 204, 215, 215, 216, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867245779Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[1]R generation[0..12]: [114, 124, 126, 128, 119, 126, 118, 126, 126, 128, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867259186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867276794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[2]R flush_numbers[0..12]: [196, 213, 215, 216, 206, 215, 204, 215, 215, 216, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867291832Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[2]R generation[0..12]: [114, 124, 126, 128, 119, 126, 118, 126, 126, 128, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.86730515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867318518Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867331887Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867345805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867358543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867371221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"":"downstairs","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.86738408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":Wait for a query_work_queue command to finish before sending IO "9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867405087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"0","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867417955Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"1","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867430543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"client":"2","":"downstairs","session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867443411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.86745535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"5856c2d2-ef69-4a56-a6c7-6a22c5f325a4 is now active with session: 9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867467868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867483126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702,"session_id":"9ca27b14-f7ef-4621-a6f3-f3d655ee4e7a"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:03.867495504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 316 len 36864 data: 13 14 14 13 13 12 13 11 12 02/30 Write block 38 len 24576 data: 10 10 11 15 17 16 03/30 Read block 97 len 8192 04/30 Read block 440 len 4096 05/30 Read block 460 len 36864 06/30 Write block 13 len 16384 data: 11 14 15 15 07/30 Read block 297 len 4096 08/30 Write block 128 len 24576 data: 16 20 19 18 16 13 09/30 Write block 181 len 16384 data: 9 11 11 10 10/30 Read block 32 len 40960 11/30 Write block 459 len 20480 data: 7 8 8 10 10 12/30 Write block 33 len 28672 data: 10 11 11 11 11 11 11 13/30 Read block 410 len 4096 14/30 Flush 15/30 Write block 297 len 16384 data: 14 16 16 14 16/30 Write block 415 len 28672 data: 12 12 12 11 10 10 11 17/30 Write block 289 len 8192 data: 15 15 18/30 Write block 371 len 12288 data: 13 10 9 19/30 Flush 20/30 Write block 437 len 28672 data: 5 4 5 6 6 7 10 21/30 Read block 411 len 4096 22/30 Read block 486 len 16384 23/30 Write block 262 len 40960 data: 9 9 12 13 13 11 11 11 14 17 24/30 Read block 40 len 12288 25/30 Read block 531 len 28672 26/30 Read block 376 len 16384 27/30 Read block 386 len 36864 28/30 Write block 423 len 12288 data: 14 11 12 29/30 Read block 209 len 16384 30/30 Write block 416 len 24576 data: 13 13 12 11 11 12 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:03.887009386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 218 218 218 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 130 130 130 218 218 218 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 130 130 130 218 218 218 F F F 7 140-159 126 126 126 215 215 215 F F F 8 160-179 126 126 126 215 215 215 F F F 9 180-199 130 130 130 218 218 218 F F F 10 200-219 129 129 129 217 217 217 F F F 11 220-239 129 129 129 217 217 217 F F F 12 240-259 128 128 128 216 216 216 F F F 13 260-279 128 128 128 216 216 216 T T T 14 280-299 130 130 130 219 219 219 F F F 15 300-319 130 130 130 219 219 219 F F F 16 320-339 130 130 130 218 218 218 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 130 130 130 219 219 219 F F F 19 380-399 129 129 129 217 217 217 F F F 20 400-419 130 130 130 219 219 219 T T T 21 420-439 130 130 130 219 219 219 T T T 22 440-459 130 130 130 218 218 218 T T T 23 460-479 130 130 130 218 218 218 F F F 24 480-499 125 125 125 214 214 214 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 128 128 128 216 216 216 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 130, Max flush: 219 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 218 218 218 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 130 130 130 218 218 218 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 119 119 119 206 206 206 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 130 130 130 218 218 218 F F F 7 140-159 126 126 126 215 215 215 F F F 8 160-179 126 126 126 215 215 215 F F F 9 180-199 130 130 130 218 218 218 F F F 10 200-219 129 129 129 217 217 217 F F F 11 220-239 129 129 129 217 217 217 F F F 12 240-259 128 128 128 216 216 216 F F F 13 260-279 131 131 131 220 220 220 F F F 14 280-299 130 130 130 219 219 219 F F F 15 300-319 130 130 130 219 219 219 F F F 16 320-339 130 130 130 218 218 218 F F F 17 340-359 121 121 121 208 208 208 F F F 18 360-379 130 130 130 219 219 219 F F F 19 380-399 129 129 129 217 217 217 F F F 20 400-419 131 131 131 220 220 220 F F F 21 420-439 131 131 131 220 220 220 F F F 22 440-459 131 131 131 220 220 220 F F F 23 460-479 130 130 130 218 218 218 F F F 24 480-499 125 125 125 214 214 214 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 128 128 128 216 216 216 F F F 27 540-559 125 125 125 214 214 214 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 124 124 124 213 213 213 F F F Max gen: 131, Max flush: 220 Begin loop 66 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.156036914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:08.200830309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.201233034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.234344917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.234892742Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.234932097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.234946904Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.234961053Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} {"msg":"Crucible d90fee79-248d-4c98-8a8a-92217a9bc251 has session id: f08b027f-5c9f-46a8-87de-be6a76b66cf5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.23548885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Upstairs opts: Upstairs UUID: d90fee79-248d-4c98-8a8a-92217a9bc251, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.235533444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Crucible stats registered with UUID: d90fee79-248d-4c98-8a8a-92217a9bc251","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.235550052Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267041876Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267072812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} {"msg":"d90fee79-248d-4c98-8a8a-92217a9bc251 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267142442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267165329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267194745Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267211863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267240289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267310569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267404446Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267437932Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267468408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267499224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_connection connected from Ok(127.0.0.1:43087)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.26752842Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_connection connected from Ok(127.0.0.1:38554)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267559105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_connection connected from Ok(127.0.0.1:48941)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.267590051Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.268145915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.268173271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.268188039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.268209087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.268233503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.2682593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.2687641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.268878735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269085956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269327164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269358659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269383186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269445427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269466224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269479643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269506259Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269520757Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[0]R flush_numbers[0..12]: [218, 218, 218, 216, 206, 215, 218, 215, 215, 218, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269538305Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[0]R generation[0..12]: [130, 130, 130, 128, 119, 126, 130, 126, 126, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269552143Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269565851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[1]R flush_numbers[0..12]: [218, 218, 218, 216, 206, 215, 218, 215, 215, 218, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269579199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[1]R generation[0..12]: [130, 130, 130, 128, 119, 126, 130, 126, 126, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269592537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269605885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[2]R flush_numbers[0..12]: [218, 218, 218, 216, 206, 215, 218, 215, 215, 218, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269622403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[2]R generation[0..12]: [130, 130, 130, 128, 119, 126, 130, 126, 126, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269636551Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269652819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269675756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269700992Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Next flush: 221","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2025-01-11T06:10:08.269731418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269770263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"":"downstairs","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269787031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269801429Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"0","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269824935Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"1","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269850272Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"client":"2","":"downstairs","session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269870029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269885837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"d90fee79-248d-4c98-8a8a-92217a9bc251 is now active with session: f08b027f-5c9f-46a8-87de-be6a76b66cf5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269908624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269931831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710,"session_id":"f08b027f-5c9f-46a8-87de-be6a76b66cf5"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:08.269956148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 82 len 12288 data: 5 6 7 03/30 Read block 326 len 28672 04/30 Write block 595 len 16384 data: 6 5 3 4 05/30 Read block 53 len 16384 06/30 Write block 132 len 20480 data: 17 14 12 13 11 07/30 Write block 1 len 28672 data: 4 5 7 10 8 9 8 08/30 Write block 539 len 28672 data: 9 10 9 9 9 12 12 09/30 Read block 481 len 28672 10/30 Write block 384 len 12288 data: 13 15 18 11/30 Read block 185 len 24576 12/30 Read block 225 len 36864 13/30 Read block 554 len 4096 14/30 Read block 339 len 40960 15/30 Write block 317 len 32768 data: 15 15 14 14 13 14 12 13 16/30 Read block 330 len 36864 17/30 Write block 247 len 8192 data: 12 12 18/30 Write block 586 len 28672 data: 7 8 8 9 10 11 10 19/30 Read block 340 len 36864 20/30 Write block 488 len 12288 data: 13 10 11 21/30 Flush 22/30 Write block 2 len 16384 data: 6 8 11 9 23/30 Read block 97 len 16384 24/30 Flush 25/30 Write block 154 len 32768 data: 8 12 13 12 11 10 11 10 26/30 Read block 56 len 12288 27/30 Read block 7 len 24576 28/30 Write block 358 len 8192 data: 12 11 29/30 Read block 321 len 36864 30/30 Write block 163 len 8192 data: 9 8 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 9 Done Done Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 2 4 Done 5 5 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:08.290069498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 130 130 130 218 218 218 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 132 132 132 222 222 222 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 132 132 132 222 222 222 F F F 7 140-159 126 126 126 215 215 215 T T T 8 160-179 126 126 126 215 215 215 T T T 9 180-199 130 130 130 218 218 218 F F F 10 200-219 129 129 129 217 217 217 F F F 11 220-239 129 129 129 217 217 217 F F F 12 240-259 132 132 132 222 222 222 F F F 13 260-279 131 131 131 220 220 220 F F F 14 280-299 130 130 130 219 219 219 F F F 15 300-319 132 132 132 222 222 222 F F F 16 320-339 132 132 132 222 222 222 F F F 17 340-359 121 121 121 208 208 208 T T T 18 360-379 130 130 130 219 219 219 F F F 19 380-399 132 132 132 222 222 222 F F F 20 400-419 131 131 131 220 220 220 F F F 21 420-439 131 131 131 220 220 220 F F F 22 440-459 131 131 131 220 220 220 F F F 23 460-479 130 130 130 218 218 218 F F F 24 480-499 132 132 132 222 222 222 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 132 132 132 222 222 222 F F F 27 540-559 132 132 132 222 222 222 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 132, Max flush: 223 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 130 130 130 218 218 218 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 132 132 132 222 222 222 F F F 5 100-119 126 126 126 215 215 215 F F F 6 120-139 132 132 132 222 222 222 F F F 7 140-159 133 133 133 224 224 224 F F F 8 160-179 133 133 133 224 224 224 F F F 9 180-199 130 130 130 218 218 218 F F F 10 200-219 129 129 129 217 217 217 F F F 11 220-239 129 129 129 217 217 217 F F F 12 240-259 132 132 132 222 222 222 F F F 13 260-279 131 131 131 220 220 220 F F F 14 280-299 130 130 130 219 219 219 F F F 15 300-319 132 132 132 222 222 222 F F F 16 320-339 132 132 132 222 222 222 F F F 17 340-359 133 133 133 224 224 224 F F F 18 360-379 130 130 130 219 219 219 F F F 19 380-399 132 132 132 222 222 222 F F F 20 400-419 131 131 131 220 220 220 F F F 21 420-439 131 131 131 220 220 220 F F F 22 440-459 131 131 131 220 220 220 F F F 23 460-479 130 130 130 218 218 218 F F F 24 480-499 132 132 132 222 222 222 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 132 132 132 222 222 222 F F F 27 540-559 132 132 132 222 222 222 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 133, Max flush: 224 Begin loop 67 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.586399812Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.633418263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634222103Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634275186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634291124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634305482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} {"msg":"Crucible dd36c3c5-b085-4f33-b9a3-cde84d93b4b6 has session id: acc196b2-f7b7-4537-8a82-24011d6f56b0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634779287Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Upstairs opts: Upstairs UUID: dd36c3c5-b085-4f33-b9a3-cde84d93b4b6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634834619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Crucible stats registered with UUID: dd36c3c5-b085-4f33-b9a3-cde84d93b4b6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.634860436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667258926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667317288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} {"msg":"dd36c3c5-b085-4f33-b9a3-cde84d93b4b6 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667347184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.66737659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667430882Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667463898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667496053Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667525839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667553736Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667571703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667589891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667606578Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:59580)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667706944Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:41535)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.667986986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:43488)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.668024581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.668573186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.668601102Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.668734224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.66876527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.668780848Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.668798395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.66935025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.669380616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.669506598Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.669903044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.669941249Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.669968875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670003281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670033117Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670059333Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670116875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670144911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[0]R flush_numbers[0..12]: [223, 218, 218, 216, 222, 215, 222, 224, 224, 218, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670160259Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[0]R generation[0..12]: [132, 130, 130, 128, 132, 126, 132, 133, 133, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670174617Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670188715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[1]R flush_numbers[0..12]: [223, 218, 218, 216, 222, 215, 222, 224, 224, 218, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670202413Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[1]R generation[0..12]: [132, 130, 130, 128, 132, 126, 132, 133, 133, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670215892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.67022989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[2]R flush_numbers[0..12]: [223, 218, 218, 216, 222, 215, 222, 224, 224, 218, 217, 217]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670243568Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[2]R generation[0..12]: [132, 130, 130, 128, 132, 126, 132, 133, 133, 130, 129, 129]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670256826Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670270304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670286652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.67030004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670312788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Wait for a query_work_queue command to finish before sending IO All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670333056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"":"downstairs","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670346374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670358322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"0","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.6703715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"1","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670384318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"client":"2","":"downstairs","session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670397257Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670409115Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"dd36c3c5-b085-4f33-b9a3-cde84d93b4b6 is now active with session: acc196b2-f7b7-4537-8a82-24011d6f56b0","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670421503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.670433132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718,"session_id":"acc196b2-f7b7-4537-8a82-24011d6f56b0"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:12.67044495Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 473 len 36864 data: 10 12 13 13 13 13 13 9 10 02/30 Read block 13 len 16384 03/30 Write block 106 len 36864 data: 11 8 9 8 8 11 10 10 10 04/30 Write block 180 len 20480 data: 10 10 12 12 11 05/30 Read block 345 len 8192 06/30 Write block 41 len 24576 data: 16 18 17 17 17 15 07/30 Write block 530 len 8192 data: 11 9 08/30 Read block 112 len 24576 09/30 Flush 10/30 Write block 444 len 40960 data: 12 14 14 13 14 15 13 11 13 15 11/30 Write block 461 len 20480 data: 9 11 11 9 8 12/30 Write block 133 len 28672 data: 15 13 14 12 9 7 7 13/30 Write block 213 len 4096 data: 12 14/30 Write block 236 len 28672 data: 13 15 14 13 12 13 11 15/30 Write block 348 len 12288 data: 11 10 7 16/30 Write block 80 len 40960 data: 7 7 6 7 8 6 7 6 7 10 17/30 Write block 372 len 4096 data: 11 18/30 Read block 408 len 28672 19/30 Write block 530 len 40960 data: 12 10 9 9 11 10 7 7 6 10 20/30 Write block 353 len 24576 data: 10 9 11 11 10 13 21/30 Read block 517 len 28672 22/30 Read block 395 len 32768 23/30 Read block 45 len 32768 24/30 Read block 517 len 36864 25/30 Write block 99 len 24576 data: 9 8 8 10 7 10 26/30 Read block 520 len 20480 27/30 Write block 424 len 28672 data: 12 13 10 11 12 11 14 28/30 Write block 279 len 8192 data: 9 6 29/30 Read block 527 len 4096 30/30 Read block 469 len 4096 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 10 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Write 10 Done Done Done false 1016 Acked Write 1 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 6 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 21 20 20 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:12.6904137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 134 134 134 225 225 225 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 132 132 132 222 222 222 T T T 5 100-119 134 134 134 225 225 225 T T T 6 120-139 132 132 132 222 222 222 T T T 7 140-159 133 133 133 224 224 224 F F F 8 160-179 133 133 133 224 224 224 F F F 9 180-199 134 134 134 225 225 225 F F F 10 200-219 129 129 129 217 217 217 T T T 11 220-239 129 129 129 217 217 217 T T T 12 240-259 132 132 132 222 222 222 T T T 13 260-279 131 131 131 220 220 220 T T T 14 280-299 130 130 130 219 219 219 T T T 15 300-319 132 132 132 222 222 222 F F F 16 320-339 132 132 132 222 222 222 F F F 17 340-359 133 133 133 224 224 224 T T T 18 360-379 130 130 130 219 219 219 T T T 19 380-399 132 132 132 222 222 222 F F F 20 400-419 131 131 131 220 220 220 F F F 21 420-439 131 131 131 220 220 220 T T T 22 440-459 131 131 131 220 220 220 T T T 23 460-479 134 134 134 225 225 225 T T T 24 480-499 134 134 134 225 225 225 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 134 134 134 225 225 225 T T T 27 540-559 132 132 132 222 222 222 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 134, Max flush: 225 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 134 134 134 225 225 225 F F F 3 060-079 128 128 128 216 216 216 F F F 4 080-099 135 135 135 226 226 226 F F F 5 100-119 135 135 135 226 226 226 F F F 6 120-139 135 135 135 226 226 226 F F F 7 140-159 133 133 133 224 224 224 F F F 8 160-179 133 133 133 224 224 224 F F F 9 180-199 134 134 134 225 225 225 F F F 10 200-219 135 135 135 226 226 226 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 135 135 135 226 226 226 F F F 13 260-279 135 135 135 226 226 226 F F F 14 280-299 135 135 135 226 226 226 F F F 15 300-319 132 132 132 222 222 222 F F F 16 320-339 132 132 132 222 222 222 F F F 17 340-359 135 135 135 226 226 226 F F F 18 360-379 135 135 135 226 226 226 F F F 19 380-399 132 132 132 222 222 222 F F F 20 400-419 131 131 131 220 220 220 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 135 135 135 226 226 226 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 225 225 225 F F F 25 500-519 126 126 126 215 215 215 F F F 26 520-539 135 135 135 226 226 226 F F F 27 540-559 132 132 132 222 222 222 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 135, Max flush: 226 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.027050943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.071954353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.07256286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.072606394Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.072620622Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.07263899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} {"msg":"Crucible 58211811-db05-46c6-8ab5-f3c260cc8691 has session id: 989203ca-849a-4b13-9f46-ba0e72c0c0e1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.073299929Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Upstairs opts: Upstairs UUID: 58211811-db05-46c6-8ab5-f3c260cc8691, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.073332655Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Crucible stats registered with UUID: 58211811-db05-46c6-8ab5-f3c260cc8691","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.073348053Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105044629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105075205Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} {"msg":"58211811-db05-46c6-8ab5-f3c260cc8691 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105142685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105165113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105193059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105210366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105226894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105248521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105279017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105308223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105335509Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105369474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:64526)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.10540008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:44422)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105430926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:61967)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.105462922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.106093295Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.106120912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.10613597Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.106149498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.10628238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.106312465Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.10679408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.106822796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.106910804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107371941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107399677Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107416945Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107429533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107443311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107456179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107468218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107481216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[0]R flush_numbers[0..12]: [223, 218, 225, 216, 226, 226, 226, 224, 224, 225, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107494534Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[0]R generation[0..12]: [132, 130, 134, 128, 135, 135, 135, 133, 133, 134, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107508042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.10752157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[1]R flush_numbers[0..12]: [223, 218, 225, 216, 226, 226, 226, 224, 224, 225, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107535048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[1]R generation[0..12]: [132, 130, 134, 128, 135, 135, 135, 133, 133, 134, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107548176Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107561765Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[2]R flush_numbers[0..12]: [223, 218, 225, 216, 226, 226, 226, 224, 224, 225, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107575083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[2]R generation[0..12]: [132, 130, 134, 128, 135, 135, 135, 133, 133, 134, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107588411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107602819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Max found gen is 136","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:10:17.107623496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107647233Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107660181Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.10767289Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"":"downstairs","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107685778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107697536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"0","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107710494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"1","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107723463Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"client":"2","":"downstairs","session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107736491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107748469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"58211811-db05-46c6-8ab5-f3c260cc8691 is now active with session: 989203ca-849a-4b13-9f46-ba0e72c0c0e1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107760918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107772366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726,"session_id":"989203ca-849a-4b13-9f46-ba0e72c0c0e1"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:17.107785444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 96 len 20480 data: 10 10 11 10 9 03/30 Write block 295 len 28672 data: 12 13 15 17 17 15 12 04/30 Read block 518 len 4096 05/30 Read block 478 len 40960 06/30 Read block 205 len 32768 07/30 Write block 79 len 40960 data: 8 8 8 7 8 9 7 8 7 8 08/30 Write block 162 len 8192 data: 10 10 09/30 Read block 108 len 24576 10/30 Write block 356 len 28672 data: 12 11 14 12 13 13 11 11/30 Write block 404 len 16384 data: 5 8 9 13 12/30 Write block 384 len 16384 data: 14 16 19 16 13/30 Read block 96 len 32768 14/30 Read block 226 len 20480 15/30 Write block 245 len 28672 data: 9 12 13 13 12 14 14 16/30 Write block 381 len 20480 data: 15 13 12 15 17 17/30 Flush 18/30 Write block 56 len 28672 data: 13 14 13 15 16 15 12 19/30 Write block 514 len 28672 data: 12 10 10 9 7 6 7 20/30 Write block 440 len 12288 data: 7 7 8 21/30 Write block 274 len 12288 data: 14 11 15 22/30 Read block 569 len 32768 23/30 Read block 576 len 12288 24/30 Flush 25/30 Write block 440 len 32768 data: 8 8 9 11 13 15 15 14 26/30 Read block 39 len 12288 27/30 Write block 548 len 28672 data: 9 8 9 9 8 9 11 28/30 Read block 195 len 12288 29/30 Write block 174 len 36864 data: 14 13 12 12 11 10 11 11 13 30/30 Read block 185 len 24576 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 4 6 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:17.127367857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 136 136 136 229 229 229 F F F 3 060-079 136 136 136 229 229 229 F F F 4 080-099 136 136 136 228 228 228 F F F 5 100-119 136 136 136 228 228 228 F F F 6 120-139 135 135 135 226 226 226 F F F 7 140-159 133 133 133 224 224 224 F F F 8 160-179 136 136 136 228 228 228 T T T 9 180-199 134 134 134 225 225 225 T T T 10 200-219 135 135 135 226 226 226 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 136 136 136 228 228 228 F F F 13 260-279 136 136 136 229 229 229 F F F 14 280-299 136 136 136 228 228 228 F F F 15 300-319 136 136 136 228 228 228 F F F 16 320-339 132 132 132 222 222 222 F F F 17 340-359 136 136 136 228 228 228 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 136 136 136 228 228 228 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 136 136 136 229 229 229 T T T 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 225 225 225 F F F 25 500-519 136 136 136 229 229 229 F F F 26 520-539 136 136 136 229 229 229 F F F 27 540-559 132 132 132 222 222 222 T T T 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 136, Max flush: 229 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 130 130 130 218 218 218 F F F 2 040-059 136 136 136 229 229 229 F F F 3 060-079 136 136 136 229 229 229 F F F 4 080-099 136 136 136 228 228 228 F F F 5 100-119 136 136 136 228 228 228 F F F 6 120-139 135 135 135 226 226 226 F F F 7 140-159 133 133 133 224 224 224 F F F 8 160-179 137 137 137 230 230 230 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 135 135 135 226 226 226 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 136 136 136 228 228 228 F F F 13 260-279 136 136 136 229 229 229 F F F 14 280-299 136 136 136 228 228 228 F F F 15 300-319 136 136 136 228 228 228 F F F 16 320-339 132 132 132 222 222 222 F F F 17 340-359 136 136 136 228 228 228 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 136 136 136 228 228 228 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 137 137 137 230 230 230 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 225 225 225 F F F 25 500-519 136 136 136 229 229 229 F F F 26 520-539 136 136 136 229 229 229 F F F 27 540-559 137 137 137 230 230 230 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 137, Max flush: 230 Begin loop 69 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.401672543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.446281544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.446885191Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.446949572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.44696747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.446986957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} {"msg":"Crucible dd91bfe9-936d-48ed-b35e-996801780801 has session id: 358726b9-699b-489c-9d75-2246c4fa0fa9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.447448094Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Upstairs opts: Upstairs UUID: dd91bfe9-936d-48ed-b35e-996801780801, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.447488928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Crucible stats registered with UUID: dd91bfe9-936d-48ed-b35e-996801780801","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.447518884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479379098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479421902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} {"msg":"dd91bfe9-936d-48ed-b35e-996801780801 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479488913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479522448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479554004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479571562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479588039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479604347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479620695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479636783Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479664589Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479682966Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:40968)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479768335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64270)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.479820168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37807)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.480047317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.480499245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.48053283Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.480560177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.480587483Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.480648445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.48068182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481115721Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481144477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481160144Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.48170692Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481733566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481747904Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481760492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.48177376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481786759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481802317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481815565Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[0]R flush_numbers[0..12]: [223, 218, 229, 229, 228, 228, 226, 224, 230, 230, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481829173Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[0]R generation[0..12]: [132, 130, 136, 136, 136, 136, 135, 133, 137, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481842731Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481856539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[1]R flush_numbers[0..12]: [223, 218, 229, 229, 228, 228, 226, 224, 230, 230, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481869997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[1]R generation[0..12]: [132, 130, 136, 136, 136, 136, 135, 133, 137, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481884505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481909162Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[2]R flush_numbers[0..12]: [223, 218, 229, 229, 228, 228, 226, 224, 230, 230, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481934669Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"[2]R generation[0..12]: [132, 130, 136, 136, 136, 136, 135, 133, 137, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481957645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481979003Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.481992001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482004899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Next flush: 231","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482020127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482032905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"":"downstairs","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482045663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482057222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"0","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.4820697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"1","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482082268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"client":"2","":"downstairs","session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482094987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482106785Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"dd91bfe9-936d-48ed-b35e-996801780801 is now active with session: 358726b9-699b-489c-9d75-2246c4fa0fa9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482119103Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.482130592Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734,"session_id":"358726b9-699b-489c-9d75-2246c4fa0fa9"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:21.48214234Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 32768 data: 6 9 11 9 10 13 12 14 02/30 Read block 264 len 32768 03/30 Read block 75 len 32768 04/30 Read block 43 len 40960 05/30 Read block 249 len 32768 06/30 Write block 39 len 8192 data: 12 12 07/30 Read block 133 len 16384 08/30 Read block 256 len 20480 09/30 Write block 393 len 24576 data: 13 11 11 9 12 11 10/30 Write block 544 len 8192 data: 13 13 11/30 Write block 259 len 28672 data: 11 9 8 10 10 13 14 12/30 Read block 455 len 36864 13/30 Flush 14/30 Write block 319 len 4096 data: 15 15/30 Read block 46 len 40960 16/30 Flush 17/30 Write block 172 len 28672 data: 15 17 15 14 13 13 12 18/30 Flush 19/30 Write block 117 len 32768 data: 8 7 6 8 8 9 6 6 20/30 Read block 82 len 16384 21/30 Flush 22/30 Write block 552 len 4096 data: 9 23/30 Read block 550 len 24576 24/30 Read block 526 len 20480 25/30 Write block 251 len 12288 data: 15 13 12 26/30 Read block 144 len 36864 27/30 Write block 346 len 8192 data: 9 10 28/30 Read block 40 len 4096 29/30 Write block 314 len 36864 data: 12 11 14 16 16 16 15 14 15 30/30 Write block 260 len 8192 data: 10 9 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:21.501096798Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 138 138 138 231 231 231 F F F 2 040-059 138 138 138 231 231 231 F F F 3 060-079 136 136 136 229 229 229 F F F 4 080-099 136 136 136 228 228 228 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 138 138 138 234 234 234 F F F 7 140-159 133 133 133 224 224 224 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 135 135 135 226 226 226 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 138 138 138 231 231 231 T T T 13 260-279 138 138 138 231 231 231 T T T 14 280-299 136 136 136 228 228 228 F F F 15 300-319 138 138 138 232 232 232 T T T 16 320-339 138 138 138 231 231 231 T T T 17 340-359 136 136 136 228 228 228 T T T 18 360-379 136 136 136 228 228 228 F F F 19 380-399 138 138 138 231 231 231 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 137 137 137 230 230 230 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 225 225 225 F F F 25 500-519 136 136 136 229 229 229 F F F 26 520-539 136 136 136 229 229 229 F F F 27 540-559 138 138 138 231 231 231 T T T 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 138, Max flush: 234 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 138 138 138 231 231 231 F F F 2 040-059 138 138 138 231 231 231 F F F 3 060-079 136 136 136 229 229 229 F F F 4 080-099 136 136 136 228 228 228 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 138 138 138 234 234 234 F F F 7 140-159 133 133 133 224 224 224 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 135 135 135 226 226 226 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 136 136 136 228 228 228 F F F 15 300-319 139 139 139 235 235 235 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 139 139 139 235 235 235 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 138 138 138 231 231 231 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 137 137 137 230 230 230 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 225 225 225 F F F 25 500-519 136 136 136 229 229 229 F F F 26 520-539 136 136 136 229 229 229 F F F 27 540-559 139 139 139 235 235 235 F F F 28 560-579 128 128 128 216 216 216 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 139, Max flush: 235 Begin loop 70 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.783103701Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.828357203Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.8289645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.829014073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.82903419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.829054358Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} {"msg":"Crucible f0533130-9c1e-41cc-b827-b9f67291597f has session id: 9d8112df-d164-4520-94bd-7fb0073c291f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.829651806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Upstairs opts: Upstairs UUID: f0533130-9c1e-41cc-b827-b9f67291597f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.82969609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Crucible stats registered with UUID: f0533130-9c1e-41cc-b827-b9f67291597f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.829713997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.860811385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.860844291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} {"msg":"f0533130-9c1e-41cc-b827-b9f67291597f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.860929189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.860951386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.860981032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.86099823Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861014898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861031275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861047653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861063541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861080279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861096777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45552)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861131982Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_connection connected from Ok(127.0.0.1:51329)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861229248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49634)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861317516Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861792931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861831636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.861979496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.86202315Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.862109808Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.862138224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.862488716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.862521042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.862614029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863063488Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863085965Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863099503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863188571Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863207818Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863220936Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863232944Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863246103Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[0]R flush_numbers[0..12]: [223, 231, 231, 229, 228, 234, 234, 224, 233, 230, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863259731Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[0]R generation[0..12]: [132, 138, 138, 136, 136, 138, 138, 133, 138, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863276319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863290446Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[1]R flush_numbers[0..12]: [223, 231, 231, 229, 228, 234, 234, 224, 233, 230, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863303975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[1]R generation[0..12]: [132, 138, 138, 136, 136, 138, 138, 133, 138, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863317363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863330891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[2]R flush_numbers[0..12]: [223, 231, 231, 229, 228, 234, 234, 224, 233, 230, 226, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863344519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[2]R generation[0..12]: [132, 138, 138, 136, 136, 138, 138, 133, 138, 137, 135, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863358127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863371805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863384314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863397252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:10:25.86340986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863427068Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"":"downstairs","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863440046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863451875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"0","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863466342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"1","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863479271Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"client":"2","":"downstairs","session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863492049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863503897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"f0533130-9c1e-41cc-b827-b9f67291597f is now active with session: 9d8112df-d164-4520-94bd-7fb0073c291f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863516316Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863528004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742,"session_id":"9d8112df-d164-4520-94bd-7fb0073c291f"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:25.863540002Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 218 len 8192 data: 12 12 02/30 Flush 03/30 Write block 28 len 8192 data: 11 10 04/30 Read block 315 len 20480 05/30 Write block 371 len 36864 data: 14 12 10 11 13 12 14 17 15 06/30 Read block 48 len 28672 07/30 Write block 288 len 24576 data: 14 16 16 16 16 16 08/30 Read block 447 len 4096 09/30 Write block 532 len 16384 data: 10 10 12 11 10/30 Write block 21 len 32768 data: 11 10 9 8 7 7 8 12 11/30 Read block 92 len 28672 12/30 Read block 300 len 12288 13/30 Read block 127 len 12288 14/30 Read block 454 len 40960 15/30 Write block 139 len 20480 data: 8 8 8 10 12 16/30 Read block 208 len 28672 17/30 Read block 11 len 40960 18/30 Flush 19/30 Write block 391 len 28672 data: 14 12 14 12 12 10 13 20/30 Read block 446 len 24576 21/30 Read block 8 len 16384 22/30 Write block 70 len 8192 data: 7 7 23/30 Write block 551 len 32768 data: 10 10 10 12 7 7 7 7 24/30 Read block 480 len 36864 25/30 Read block 210 len 16384 26/30 Write block 576 len 16384 data: 9 8 9 8 27/30 Write block 490 len 20480 data: 12 12 13 15 14 28/30 Read block 564 len 24576 29/30 Write block 552 len 4096 data: 11 30/30 Write block 360 len 32768 data: 14 14 12 11 10 10 11 14 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 2 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:25.883851695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 138 138 138 231 231 231 F F F 3 060-079 136 136 136 229 229 229 T T T 4 080-099 136 136 136 228 228 228 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 140 140 140 237 237 237 F F F 7 140-159 140 140 140 237 237 237 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 140 140 140 236 236 236 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 139 139 139 235 235 235 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 139 139 139 235 235 235 F F F 18 360-379 140 140 140 237 237 237 T T T 19 380-399 138 138 138 231 231 231 T T T 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 137 137 137 230 230 230 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 134 134 134 225 225 225 T T T 25 500-519 136 136 136 229 229 229 F F F 26 520-539 140 140 140 237 237 237 F F F 27 540-559 139 139 139 235 235 235 T T T 28 560-579 128 128 128 216 216 216 T T T 29 580-599 132 132 132 222 222 222 F F F Max gen: 140, Max flush: 237 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 223 223 223 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 138 138 138 231 231 231 F F F 3 060-079 141 141 141 238 238 238 F F F 4 080-099 136 136 136 228 228 228 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 140 140 140 237 237 237 F F F 7 140-159 140 140 140 237 237 237 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 140 140 140 236 236 236 F F F 11 220-239 135 135 135 226 226 226 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 139 139 139 235 235 235 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 139 139 139 235 235 235 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 139 139 139 235 235 235 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 141 141 141 238 238 238 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 137 137 137 230 230 230 F F F 23 460-479 135 135 135 226 226 226 F F F 24 480-499 141 141 141 238 238 238 F F F 25 500-519 136 136 136 229 229 229 F F F 26 520-539 140 140 140 237 237 237 F F F 27 540-559 141 141 141 238 238 238 F F F 28 560-579 141 141 141 238 238 238 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 141, Max flush: 238 Begin loop 71 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.168313433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:30.21265455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.213044677Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.245666107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.246229719Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.246264545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.246278703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.246292681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} {"msg":"Crucible 4dc02651-aa1f-481f-8e0f-9de444d984a4 has session id: 949ad59c-4b0e-4955-8a38-f4ad04f4e8e9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.246827718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Upstairs opts: Upstairs UUID: 4dc02651-aa1f-481f-8e0f-9de444d984a4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.246870172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Crucible stats registered with UUID: 4dc02651-aa1f-481f-8e0f-9de444d984a4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.24688749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.27869659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278726926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} {"msg":"4dc02651-aa1f-481f-8e0f-9de444d984a4 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278790167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278828402Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278861398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278893624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278924329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278949696Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.278979302Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279013167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279044033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279073809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58097)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279171195Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61807)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279266133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33614)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279289189Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.27994321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279971636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.279987104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.280004601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.280018799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.280032618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.280586172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.280621097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.28074773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281171052Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281199338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281214286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281226505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281239523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281252481Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281370855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281390362Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[0]R flush_numbers[0..12]: [223, 237, 231, 238, 228, 234, 237, 237, 233, 230, 236, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.28140448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[0]R generation[0..12]: [132, 140, 138, 141, 136, 138, 140, 140, 138, 137, 140, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281422628Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281436906Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[1]R flush_numbers[0..12]: [223, 237, 231, 238, 228, 234, 237, 237, 233, 230, 236, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281450414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[1]R generation[0..12]: [132, 140, 138, 141, 136, 138, 140, 140, 138, 137, 140, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281463722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.28147731Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[2]R flush_numbers[0..12]: [223, 237, 231, 238, 228, 234, 237, 237, 233, 230, 236, 226]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281490748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[2]R generation[0..12]: [132, 140, 138, 141, 136, 138, 140, 140, 138, 137, 140, 135]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281504056Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281517455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281529913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281543561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Next flush: 239","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281556159Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281568898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"":"downstairs","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281581706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {Wait for a query_work_queue command to finish before sending IO "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281601383Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"0","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281614341Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"1","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.28162855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"client":"2","":"downstairs","session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281642737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281654766Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"4dc02651-aa1f-481f-8e0f-9de444d984a4 is now active with session: 949ad59c-4b0e-4955-8a38-f4ad04f4e8e9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281667264Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281678943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750,"session_id":"949ad59c-4b0e-4955-8a38-f4ad04f4e8e9"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:30.281690761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 308 len 16384 data: 6 5 7 7 02/30 Write block 341 len 12288 data: 17 14 16 03/30 Read block 446 len 28672 04/30 Read block 106 len 28672 05/30 Read block 148 len 28672 06/30 Write block 554 len 36864 data: 13 8 8 8 8 7 10 11 11 07/30 Write block 41 len 32768 data: 17 19 18 18 18 16 13 14 08/30 Read block 381 len 12288 09/30 Write block 546 len 16384 data: 10 11 10 9 10/30 Flush 11/30 Write block 15 len 12288 data: 16 16 13 12/30 Write block 72 len 24576 data: 7 7 7 6 7 6 13/30 Write block 213 len 36864 data: 13 10 9 13 14 13 13 12 10 14/30 Write block 8 len 32768 data: 10 11 12 12 11 12 15 17 15/30 Read block 313 len 4096 16/30 Write block 82 len 20480 data: 8 9 10 8 9 17/30 Write block 517 len 32768 data: 10 8 7 8 8 10 11 10 18/30 Write block 458 len 4096 data: 10 19/30 Read block 272 len 4096 20/30 Read block 559 len 32768 21/30 Write block 271 len 36864 data: 18 15 13 15 12 16 15 13 10 22/30 Read block 45 len 24576 23/30 Read block 63 len 20480 24/30 Flush 25/30 Flush 26/30 Flush 27/30 Write block 479 len 16384 data: 14 10 11 10 28/30 Read block 495 len 8192 29/30 Write block 503 len 12288 data: 6 7 9 30/30 Write block 483 len 32768 data: 10 8 10 12 11 14 11 13 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 4 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 3 Sent Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 2 2 3 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:30.300604095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 142 142 142 239 239 239 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 140 140 140 237 237 237 F F F 7 140-159 140 140 140 237 237 237 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 142 142 142 240 240 240 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 142 142 142 240 240 240 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 142 142 142 239 239 239 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 142 142 142 239 239 239 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 141 141 141 238 238 238 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 135 135 135 226 226 226 T T T 24 480-499 141 141 141 238 238 238 T T T 25 500-519 142 142 142 240 240 240 T T T 26 520-539 142 142 142 240 240 240 F F F 27 540-559 142 142 142 239 239 239 F F F 28 560-579 142 142 142 239 239 239 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 142, Max flush: 240 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 142 142 142 239 239 239 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 138 138 138 234 234 234 F F F 6 120-139 140 140 140 237 237 237 F F F 7 140-159 140 140 140 237 237 237 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 142 142 142 240 240 240 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 142 142 142 240 240 240 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 142 142 142 239 239 239 F F F 16 320-339 139 139 139 235 235 235 F F F 17 340-359 142 142 142 239 239 239 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 141 141 141 238 238 238 F F F 20 400-419 136 136 136 228 228 228 F F F 21 420-439 135 135 135 226 226 226 F F F 22 440-459 142 142 142 240 240 240 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 143 143 143 241 241 241 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 142 142 142 240 240 240 F F F 27 540-559 142 142 142 239 239 239 F F F 28 560-579 142 142 142 239 239 239 F F F 29 580-599 132 132 132 222 222 222 F F F Max gen: 143, Max flush: 241 Begin loop 72 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.600867672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.646866762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.64768439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.647741303Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.6477606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.647778507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} {"msg":"Crucible 1abbf642-b9d4-4341-add1-89b6f0736c29 has session id: e362de28-d1ec-4ead-bf1c-a00b9232abe8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.648431048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Upstairs opts: Upstairs UUID: 1abbf642-b9d4-4341-add1-89b6f0736c29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.648477432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Crucible stats registered with UUID: 1abbf642-b9d4-4341-add1-89b6f0736c29","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.648501589Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.680918646Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68096756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} {"msg":"1abbf642-b9d4-4341-add1-89b6f0736c29 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681042039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681090913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681150264Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681194609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681237363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681281247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681303104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681341158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681373364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681407349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_connection connected from Ok(127.0.0.1:48834)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681433376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_connection connected from Ok(127.0.0.1:52108)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681469111Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_connection connected from Ok(127.0.0.1:37982)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.681495387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.682317085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68235305Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.682386435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.682421171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.682446777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.682480643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.683102828Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.683161749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.683210673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.683790123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68381749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.683840147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.683984537Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.6840331Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684072185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684102991Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684136516Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[0]R flush_numbers[0..12]: [240, 237, 239, 240, 240, 234, 237, 237, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684160083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[0]R generation[0..12]: [142, 140, 142, 142, 142, 138, 140, 140, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68418173Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684205317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[1]R flush_numbers[0..12]: [240, 237, 239, 240, 240, 234, 237, 237, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684229803Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[1]R generation[0..12]: [142, 140, 142, 142, 142, 138, 140, 140, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68425667Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684285896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[2]R flush_numbers[0..12]: [240, 237, 239, 240, 240, 234, 237, 237, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684313412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[2]R generation[0..12]: [142, 140, 142, 142, 142, 138, 140, 140, 138, 137, 142, 142]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:10:34.684341908Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684370734Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684396491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684422687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684448784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.6844747Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"":"downstairs","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684504626Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684524393Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"0","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684538201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"1","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68455153Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"client":"2","":"downstairs","session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684564748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684576566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"1abbf642-b9d4-4341-add1-89b6f0736c29 is now active with session: e362de28-d1ec-4ead-bf1c-a00b9232abe8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684592594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.684604392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758,"session_id":"e362de28-d1ec-4ead-bf1c-a00b9232abe8"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:34.68461645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 46 len 36864 data: 17 14 15 12 9 8 7 7 9 02/30 Read block 412 len 24576 03/30 Read block 424 len 32768 04/30 Write block 429 len 16384 data: 12 15 10 9 05/30 Read block 567 len 20480 06/30 Write block 397 len 28672 data: 14 12 10 8 8 7 6 07/30 Write block 308 len 40960 data: 7 6 8 8 6 8 13 12 15 17 08/30 Write block 142 len 28672 data: 11 13 14 15 13 13 13 09/30 Read block 85 len 36864 10/30 Flush 11/30 Write block 351 len 36864 data: 8 8 11 10 12 13 12 15 13 12/30 Write block 443 len 12288 data: 12 14 16 13/30 Write block 324 len 4096 data: 14 14/30 Write block 105 len 24576 data: 12 12 9 10 9 9 15/30 Write block 538 len 40960 data: 7 11 11 10 10 10 14 14 11 12 16/30 Write block 543 len 16384 data: 11 15 15 12 17/30 Read block 561 len 28672 18/30 Write block 588 len 40960 data: 9 10 11 12 11 10 8 7 6 4 19/30 Read block 358 len 16384 20/30 Write block 312 len 28672 data: 7 9 14 13 16 18 17 21/30 Write block 134 len 8192 data: 14 15 22/30 Read block 303 len 8192 23/30 Flush 24/30 Write block 407 len 40960 data: 14 14 14 13 13 16 13 13 13 14 25/30 Read block 250 len 32768 26/30 Read block 253 len 12288 27/30 Write block 415 len 4096 data: 14 28/30 Read block 256 len 28672 29/30 Read block 222 len 32768 30/30 Read block 337 len 24576 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 7 7 6 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:34.704642223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 144 144 144 242 242 242 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 144 144 144 243 243 243 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 142 142 142 240 240 240 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 142 142 142 240 240 240 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 144 144 144 243 243 243 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 144 144 144 243 243 243 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 144 144 144 242 242 242 F F F 20 400-419 144 144 144 242 242 242 T T T 21 420-439 144 144 144 242 242 242 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 143 143 143 241 241 241 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 144 144 144 243 243 243 F F F 27 540-559 144 144 144 243 243 243 F F F 28 560-579 142 142 142 239 239 239 F F F 29 580-599 144 144 144 243 243 243 F F F Max gen: 144, Max flush: 243 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 144 144 144 242 242 242 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 142 142 142 240 240 240 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 144 144 144 243 243 243 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 142 142 142 240 240 240 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 139 139 139 235 235 235 F F F 13 260-279 142 142 142 240 240 240 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 144 144 144 243 243 243 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 144 144 144 243 243 243 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 144 144 144 242 242 242 F F F 20 400-419 145 145 145 244 244 244 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 143 143 143 241 241 241 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 144 144 144 243 243 243 F F F 27 540-559 144 144 144 243 243 243 F F F 28 560-579 142 142 142 239 239 239 F F F 29 580-599 144 144 144 243 243 243 F F F Max gen: 145, Max flush: 244 Begin loop 73 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:38.961863713Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.007473187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008086383Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008126268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008144855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008159723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} {"msg":"Crucible b6e7bbb6-b07e-47ef-adc8-dbec04012177 has session id: b977194e-304e-42e0-8669-a5f8e89349e7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008605962Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Upstairs opts: Upstairs UUID: b6e7bbb6-b07e-47ef-adc8-dbec04012177, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008649846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Crucible stats registered with UUID: b6e7bbb6-b07e-47ef-adc8-dbec04012177","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.008674703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040469186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040504291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} {"msg":"b6e7bbb6-b07e-47ef-adc8-dbec04012177 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040574681Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040596988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040625884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040642762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04065938Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040678457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040699814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040733029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040762796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040791561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43112)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040822347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60057)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.040852773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:34760)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04087884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04168175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.041710486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.041725904Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.041739762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04175276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.041773867Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.042393122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.042423109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.042439036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043009558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043048403Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043074599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043095666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043119083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04314254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043162447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043183514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[0]R flush_numbers[0..12]: [240, 237, 242, 240, 240, 243, 243, 242, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043206002Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[0]R generation[0..12]: [142, 140, 144, 142, 142, 144, 144, 144, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043230238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043256424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[1]R flush_numbers[0..12]: [240, 237, 242, 240, 240, 243, 243, 242, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043280181Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[1]R generation[0..12]: [142, 140, 144, 142, 142, 144, 144, 144, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043303818Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043330854Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R flush_numbers[0..12]: [240, 237, 242, 240, 240, 243, 243, 242, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04336555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[2]R generation[0..12]: [142, 140, 144, 142, 142, 144, 144, 144, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043398385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043427761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043454517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043481044Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Next flush: 245","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04350693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043531397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"":"downstairs","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043556654Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.04358353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"0","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043605447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"1","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043619605Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"client":"2","":"downstairs","session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043632933Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043645941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"b6e7bbb6-b07e-47ef-adc8-dbec04012177 is now active with session: b977194e-304e-42e0-8669-a5f8e89349e7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043658549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043670128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766,"session_id":"b977194e-304e-42e0-8669-a5f8e89349e7"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:39.043682116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 60 len 24576 data: 17 16 13 12 11 8 02/30 Read block 49 len 12288 03/30 Flush 04/30 Write block 68 len 28672 data: 7 7 8 8 8 8 8 05/30 Read block 118 len 16384 06/30 Write block 392 len 24576 data: 13 15 13 13 11 15 07/30 Write block 67 len 20480 data: 10 8 8 9 9 08/30 Read block 375 len 24576 09/30 Read block 249 len 16384 10/30 Read block 112 len 4096 11/30 Write block 490 len 8192 data: 14 13 12/30 Read block 207 len 32768 13/30 Read block 53 len 8192 14/30 Write block 306 len 24576 data: 8 7 8 7 9 9 15/30 Read block 145 len 36864 16/30 Read block 327 len 40960 17/30 Write block 117 len 32768 data: 9 8 7 9 9 10 7 7 18/30 Read block 77 len 28672 19/30 Read block 373 len 4096 20/30 Read block 291 len 12288 21/30 Write block 242 len 16384 data: 12 11 11 10 22/30 Read block 332 len 20480 23/30 Flush 24/30 Write block 93 len 36864 data: 11 11 11 11 11 12 11 10 9 25/30 Read block 15 len 32768 26/30 Write block 534 len 32768 data: 13 12 8 8 8 12 12 11 27/30 Read block 193 len 20480 28/30 Write block 565 len 24576 data: 10 9 9 9 9 7 29/30 Write block 257 len 8192 data: 12 13 30/30 Read block 2 len 36864 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 2 Sent Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 5 6 7 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:39.063717347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 144 144 144 242 242 242 F F F 3 060-079 146 146 146 246 246 246 F F F 4 080-099 142 142 142 240 240 240 T T T 5 100-119 146 146 146 246 246 246 T T T 6 120-139 146 146 146 246 246 246 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 142 142 142 240 240 240 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 146 146 146 246 246 246 T T T 13 260-279 142 142 142 240 240 240 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 144 144 144 243 243 243 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 146 146 146 246 246 246 F F F 20 400-419 145 145 145 244 244 244 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 146 146 146 246 246 246 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 144 144 144 243 243 243 T T T 27 540-559 144 144 144 243 243 243 T T T 28 560-579 142 142 142 239 239 239 T T T 29 580-599 144 144 144 243 243 243 F F F Max gen: 146, Max flush: 246 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 144 144 144 242 242 242 F F F 3 060-079 146 146 146 246 246 246 F F F 4 080-099 147 147 147 247 247 247 F F F 5 100-119 147 147 147 247 247 247 F F F 6 120-139 146 146 146 246 246 246 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 137 137 137 230 230 230 F F F 10 200-219 142 142 142 240 240 240 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 147 147 147 247 247 247 F F F 13 260-279 142 142 142 240 240 240 F F F 14 280-299 140 140 140 237 237 237 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 144 144 144 243 243 243 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 146 146 146 246 246 246 F F F 20 400-419 145 145 145 244 244 244 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 144 144 144 243 243 243 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 146 146 146 246 246 246 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 147 147 147 247 247 247 F F F 27 540-559 147 147 147 247 247 247 F F F 28 560-579 147 147 147 247 247 247 F F F 29 580-599 144 144 144 243 243 243 F F F Max gen: 147, Max flush: 247 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.344103461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.388982105Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.389692898Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.38974768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.389766138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.389781835Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} {"msg":"Crucible 1654ebcc-fe22-4284-ae71-ef3e60f471b8 has session id: db175676-a2c8-4c9c-89c1-b7436d86f7c6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.390440435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Upstairs opts: Upstairs UUID: 1654ebcc-fe22-4284-ae71-ef3e60f471b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.390484969Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Crucible stats registered with UUID: 1654ebcc-fe22-4284-ae71-ef3e60f471b8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.390512726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421621442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421652328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} {"msg":"1654ebcc-fe22-4284-ae71-ef3e60f471b8 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421717049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421748085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421790559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421825424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421861079Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421892405Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421924651Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421957216Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.421991122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422024847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61761)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422058452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:50309)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.42214465Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_connection connected from Ok(127.0.0.1:61957)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422186375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.42273091Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422759217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422788522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.42280343Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422828067Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.422842375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.423353875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.423395179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.423502135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.423914948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.423941075Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.423954723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424033732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424057959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424072167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424083935Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424096883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[0]R flush_numbers[0..12]: [240, 237, 242, 246, 247, 247, 246, 242, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424110752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[0]R generation[0..12]: [142, 140, 144, 146, 147, 147, 146, 144, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.42412436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424138488Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[1]R flush_numbers[0..12]: [240, 237, 242, 246, 247, 247, 246, 242, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424152136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[1]R generation[0..12]: [142, 140, 144, 146, 147, 147, 146, 144, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424165524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424179032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[2]R flush_numbers[0..12]: [240, 237, 242, 246, 247, 247, 246, 242, 233, 230, 240, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424192321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[2]R generation[0..12]: [142, 140, 144, 146, 147, 147, 146, 144, 138, 137, 142, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424205559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424219007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424238414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424253452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Next flush: 248","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.42426618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424279138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"":"downstairs","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424292087Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424303795Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"0","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424316504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"1","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424329142Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"client":"2","":"downstairs","session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.42434185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424353748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"1654ebcc-fe22-4284-ae71-ef3e60f471b8 is now active with session: db175676-a2c8-4c9c-89c1-b7436d86f7c6","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424366157Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424377735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774,"session_id":"db175676-a2c8-4c9c-89c1-b7436d86f7c6"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:43.424389574Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 297 len 4096 data: 16 03/30 Read block 260 len 32768 04/30 Write block 133 len 12288 data: 16 15 16 05/30 Read block 287 len 36864 06/30 Read block 324 len 8192 07/30 Write block 472 len 12288 data: 9 11 13 08/30 Write block 264 len 20480 data: 14 15 14 12 12 09/30 Write block 524 len 36864 data: 11 11 11 13 13 12 13 11 11 10/30 Read block 263 len 20480 11/30 Write block 197 len 36864 data: 11 10 10 10 9 8 8 6 9 12/30 Read block 206 len 32768 13/30 Write block 290 len 8192 data: 17 17 14/30 Read block 278 len 32768 15/30 Read block 231 len 4096 16/30 Read block 449 len 32768 17/30 Write block 344 len 28672 data: 13 14 10 11 12 11 8 18/30 Flush 19/30 Write block 380 len 12288 data: 15 16 14 20/30 Read block 40 len 24576 21/30 Write block 72 len 32768 data: 9 9 9 7 8 7 8 9 22/30 Read block 176 len 8192 23/30 Write block 491 len 28672 data: 14 14 16 15 15 13 14 24/30 Write block 442 len 36864 data: 10 13 15 17 16 15 15 16 14 25/30 Read block 379 len 32768 26/30 Write block 248 len 4096 data: 14 27/30 Write block 551 len 24576 data: 11 12 11 14 9 9 28/30 Read block 504 len 32768 29/30 Write block 591 len 28672 data: 13 12 11 9 8 7 5 30/30 Write block 46 len 8192 data: 18 15 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:43.443190233Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 144 144 144 242 242 242 T T T 3 060-079 146 146 146 246 246 246 T T T 4 080-099 147 147 147 247 247 247 F F F 5 100-119 147 147 147 247 247 247 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 148 148 148 249 249 249 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 147 147 147 247 247 247 T T T 13 260-279 148 148 148 249 249 249 F F F 14 280-299 148 148 148 249 249 249 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 146 146 146 246 246 246 T T T 20 400-419 145 145 145 244 244 244 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 144 144 144 243 243 243 T T T 23 460-479 148 148 148 249 249 249 F F F 24 480-499 146 146 146 246 246 246 T T T 25 500-519 143 143 143 241 241 241 F F F 26 520-539 148 148 148 249 249 249 F F F 27 540-559 147 147 147 247 247 247 T T T 28 560-579 147 147 147 247 247 247 F F F 29 580-599 144 144 144 243 243 243 T T T Max gen: 148, Max flush: 249 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 142 142 142 240 240 240 F F F 1 020-039 140 140 140 237 237 237 F F F 2 040-059 149 149 149 250 250 250 F F F 3 060-079 149 149 149 250 250 250 F F F 4 080-099 147 147 147 247 247 247 F F F 5 100-119 147 147 147 247 247 247 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 F F F 9 180-199 148 148 148 249 249 249 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 142 142 142 240 240 240 F F F 12 240-259 149 149 149 250 250 250 F F F 13 260-279 148 148 148 249 249 249 F F F 14 280-299 148 148 148 249 249 249 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 149 149 149 250 250 250 F F F 20 400-419 145 145 145 244 244 244 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 149 149 149 250 250 250 F F F 23 460-479 148 148 148 249 249 249 F F F 24 480-499 149 149 149 250 250 250 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 148 148 148 249 249 249 F F F 27 540-559 149 149 149 250 250 250 F F F 28 560-579 147 147 147 247 247 247 F F F 29 580-599 149 149 149 250 250 250 F F F Max gen: 149, Max flush: 250 Begin loop 75 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.737944424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:47.782148279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.782533326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.815268081Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.815840602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.815881707Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.815895855Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.815909883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} {"msg":"Crucible 2e746adb-ed5b-4ca2-93e8-da5a7cecfb67 has session id: cca05ec6-3472-474c-8d32-334ef5a6f831","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.816507901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Upstairs opts: Upstairs UUID: 2e746adb-ed5b-4ca2-93e8-da5a7cecfb67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.816552485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Crucible stats registered with UUID: 2e746adb-ed5b-4ca2-93e8-da5a7cecfb67","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.816569652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847590961Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847619647Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} {"msg":"2e746adb-ed5b-4ca2-93e8-da5a7cecfb67 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847692127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847713284Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847750009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847782455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.84781422Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847855755Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.84788813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847918066Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847941683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847959211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_connection connected from Ok(127.0.0.1:42515)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.847976238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_connection connected from Ok(127.0.0.1:55662)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.848191699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_connection connected from Ok(127.0.0.1:63227)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.848230643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.848707718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.848735025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.848859777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.848879475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849043143Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849069199Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849373817Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849443738Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849654239Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849912504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.84993655Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849950738Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.849990413Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.85001179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850025768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.85015847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850178047Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[0]R flush_numbers[0..12]: [240, 237, 250, 250, 247, 247, 249, 242, 233, 249, 249, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850192255Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[0]R generation[0..12]: [142, 140, 149, 149, 147, 147, 148, 144, 138, 148, 148, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850206084Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850220211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[1]R flush_numbers[0..12]: [240, 237, 250, 250, 247, 247, 249, 242, 233, 249, 249, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.85023381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[1]R generation[0..12]: [142, 140, 149, 149, 147, 147, 148, 144, 138, 148, 148, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850247398Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850261116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[2]R flush_numbers[0..12]: [240, 237, 250, 250, 247, 247, 249, 242, 233, 249, 249, 240]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850274624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[2]R generation[0..12]: [142, 140, 149, 149, 147, 147, 148, 144, 138, 148, 148, 142]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850288042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.8503017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850314379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850330076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850343045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {Wait for a query_work_queue command to finish before sending IO "msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850362252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"":"downstairs","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.85037546Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850387379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"0","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850400277Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"1","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850413035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"client":"2","":"downstairs","session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850426133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850438202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"2e746adb-ed5b-4ca2-93e8-da5a7cecfb67 is now active with session: cca05ec6-3472-474c-8d32-334ef5a6f831","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.85045086Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850462668Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782,"session_id":"cca05ec6-3472-474c-8d32-334ef5a6f831"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:47.850474897Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 521 len 32768 data: 9 11 12 12 12 12 14 14 02/30 Read block 516 len 16384 03/30 Read block 596 len 12288 04/30 Write block 469 len 4096 data: 8 05/30 Read block 462 len 28672 06/30 Read block 309 len 16384 07/30 Read block 533 len 28672 08/30 Write block 14 len 16384 data: 16 18 17 14 09/30 Read block 186 len 28672 10/30 Flush 11/30 Write block 79 len 24576 data: 10 9 9 9 10 11 12/30 Flush 13/30 Flush 14/30 Write block 539 len 12288 data: 13 13 12 15/30 Flush 16/30 Write block 232 len 4096 data: 9 17/30 Write block 17 len 16384 data: 15 14 14 13 18/30 Write block 533 len 32768 data: 11 14 13 9 9 9 14 14 19/30 Flush 20/30 Write block 411 len 8192 data: 14 17 21/30 Read block 123 len 16384 22/30 Read block 589 len 36864 23/30 Read block 125 len 16384 24/30 Read block 393 len 8192 25/30 Write block 541 len 40960 data: 13 11 12 16 16 13 13 11 10 10 26/30 Flush 27/30 Write block 172 len 8192 data: 16 18 28/30 Read block 204 len 28672 29/30 Read block 560 len 4096 30/30 Write block 0 len 28672 data: 3 5 7 9 12 10 10 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:47.869521523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 150 150 150 255 255 255 T T T 1 020-039 150 150 150 255 255 255 F F F 2 040-059 149 149 149 250 250 250 F F F 3 060-079 150 150 150 252 252 252 F F F 4 080-099 150 150 150 252 252 252 F F F 5 100-119 147 147 147 247 247 247 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 138 138 138 233 233 233 T T T 9 180-199 148 148 148 249 249 249 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 150 150 150 255 255 255 F F F 12 240-259 149 149 149 250 250 250 F F F 13 260-279 148 148 148 249 249 249 F F F 14 280-299 148 148 148 249 249 249 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 149 149 149 250 250 250 F F F 20 400-419 150 150 150 256 256 256 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 149 149 149 250 250 250 F F F 23 460-479 150 150 150 251 251 251 F F F 24 480-499 149 149 149 250 250 250 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 150 150 150 255 255 255 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 147 147 147 247 247 247 F F F 29 580-599 149 149 149 250 250 250 F F F Max gen: 150, Max flush: 256 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 151 151 151 257 257 257 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 149 149 149 250 250 250 F F F 3 060-079 150 150 150 252 252 252 F F F 4 080-099 150 150 150 252 252 252 F F F 5 100-119 147 147 147 247 247 247 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 144 144 144 242 242 242 F F F 8 160-179 151 151 151 257 257 257 F F F 9 180-199 148 148 148 249 249 249 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 150 150 150 255 255 255 F F F 12 240-259 149 149 149 250 250 250 F F F 13 260-279 148 148 148 249 249 249 F F F 14 280-299 148 148 148 249 249 249 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 149 149 149 250 250 250 F F F 20 400-419 150 150 150 256 256 256 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 149 149 149 250 250 250 F F F 23 460-479 150 150 150 251 251 251 F F F 24 480-499 149 149 149 250 250 250 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 150 150 150 255 255 255 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 147 147 147 247 247 247 F F F 29 580-599 149 149 149 250 250 250 F F F Max gen: 151, Max flush: 257 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.16371456Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.210744909Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.211404809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.211453123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.21147152Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.211490727Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} {"msg":"Crucible 0034709b-e0ec-4687-8925-d664d1648957 has session id: d6c5e8f8-d933-4cef-9a5d-92df540b4de2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.212070638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Upstairs opts: Upstairs UUID: 0034709b-e0ec-4687-8925-d664d1648957, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.212108653Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Crucible stats registered with UUID: 0034709b-e0ec-4687-8925-d664d1648957","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.212124691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243691515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.2437247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} {"msg":"0034709b-e0ec-4687-8925-d664d1648957 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243805769Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243828896Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243858742Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243889108Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243921163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243952849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.243986215Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.2440188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244053245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244085051Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_connection connected from Ok(127.0.0.1:34194)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.24423797Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_connection connected from Ok(127.0.0.1:64118)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244275705Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_connection connected from Ok(127.0.0.1:55442)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244296682Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244699737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244734582Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244762098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244793544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.244991057Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.245018093Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.245500497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.245528824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.245543511Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246086957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246114253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246128882Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246197602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.24621703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246230788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246242816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246255734Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[0]R flush_numbers[0..12]: [257, 255, 250, 252, 252, 247, 249, 242, 257, 249, 249, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246269212Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[0]R generation[0..12]: [151, 150, 149, 150, 150, 147, 148, 144, 151, 148, 148, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.24628581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246299878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[1]R flush_numbers[0..12]: [257, 255, 250, 252, 252, 247, 249, 242, 257, 249, 249, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246313346Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[1]R generation[0..12]: [151, 150, 149, 150, 150, 147, 148, 144, 151, 148, 148, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246326604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246340113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[2]R flush_numbers[0..12]: [257, 255, 250, 252, 252, 247, 249, 242, 257, 249, 249, 255]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246353421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[2]R generation[0..12]: [151, 150, 149, 150, 150, 147, 148, 144, 151, 148, 148, 150]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246366719Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246380397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246392925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246405644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Next flush: 258","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246418082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:10:52.24643067Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"":"downstairs","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246449978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246461666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"0","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246477204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"1","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246490532Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"client":"2","":"downstairs","session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.24650332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246515349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"0034709b-e0ec-4687-8925-d664d1648957 is now active with session: d6c5e8f8-d933-4cef-9a5d-92df540b4de2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246527767Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246539335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790,"session_id":"d6c5e8f8-d933-4cef-9a5d-92df540b4de2"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:52.246551154Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 230 len 20480 data: 10 10 10 10 13 02/30 Read block 594 len 12288 03/30 Read block 356 len 36864 04/30 Write block 106 len 8192 data: 13 10 05/30 Read block 77 len 8192 06/30 Write block 477 len 32768 data: 14 14 15 11 12 11 11 9 07/30 Write block 19 len 4096 data: 15 08/30 Write block 595 len 8192 data: 9 8 09/30 Read block 205 len 32768 10/30 Read block 517 len 16384 11/30 Read block 406 len 28672 12/30 Write block 469 len 40960 data: 9 8 9 10 12 14 14 14 15 15 13/30 Read block 119 len 40960 14/30 Write block 190 len 12288 data: 8 6 6 15/30 Read block 292 len 8192 16/30 Write block 151 len 40960 data: 8 7 7 9 13 14 13 12 11 12 17/30 Write block 258 len 28672 data: 14 12 11 10 11 11 15 18/30 Write block 49 len 24576 data: 13 10 9 8 8 10 19/30 Read block 42 len 16384 20/30 Read block 56 len 20480 21/30 Flush 22/30 Write block 450 len 36864 data: 15 12 14 16 10 9 9 10 11 23/30 Read block 496 len 36864 24/30 Read block 470 len 24576 25/30 Flush 26/30 Write block 244 len 24576 data: 12 11 13 14 15 13 27/30 Write block 140 len 12288 data: 9 9 12 28/30 Read block 33 len 40960 29/30 Read block 155 len 4096 30/30 Read block 340 len 20480 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 4 5 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:52.266759241Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 258 258 258 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 152 152 152 258 258 258 F F F 3 060-079 150 150 150 252 252 252 F F F 4 080-099 150 150 150 252 252 252 F F F 5 100-119 152 152 152 258 258 258 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 152 152 152 258 258 258 T T T 8 160-179 152 152 152 258 258 258 F F F 9 180-199 152 152 152 258 258 258 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 152 152 152 258 258 258 F F F 12 240-259 152 152 152 258 258 258 T T T 13 260-279 152 152 152 258 258 258 F F F 14 280-299 148 148 148 249 249 249 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 149 149 149 250 250 250 F F F 20 400-419 150 150 150 256 256 256 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 152 152 152 258 258 258 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 150 150 150 255 255 255 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 147 147 147 247 247 247 F F F 29 580-599 152 152 152 258 258 258 F F F Max gen: 152, Max flush: 259 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 258 258 258 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 152 152 152 258 258 258 F F F 3 060-079 150 150 150 252 252 252 F F F 4 080-099 150 150 150 252 252 252 F F F 5 100-119 152 152 152 258 258 258 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 152 152 152 258 258 258 F F F 9 180-199 152 152 152 258 258 258 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 152 152 152 258 258 258 F F F 12 240-259 153 153 153 260 260 260 F F F 13 260-279 152 152 152 258 258 258 F F F 14 280-299 148 148 148 249 249 249 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 F F F 19 380-399 149 149 149 250 250 250 F F F 20 400-419 150 150 150 256 256 256 F F F 21 420-439 144 144 144 242 242 242 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 152 152 152 258 258 258 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 150 150 150 255 255 255 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 147 147 147 247 247 247 F F F 29 580-599 152 152 152 258 258 258 F F F Max gen: 153, Max flush: 260 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.522600421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.567458778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.568072104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.568124987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.568143724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.568161612Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} {"msg":"Crucible 034e186d-d8af-4bc6-8b56-c74fe923e02e has session id: 6e085d5e-af25-4766-af6a-501422176e5c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.568658454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Upstairs opts: Upstairs UUID: 034e186d-d8af-4bc6-8b56-c74fe923e02e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.56875951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Crucible stats registered with UUID: 034e186d-d8af-4bc6-8b56-c74fe923e02e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.568791485Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.599925399Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.599957094Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} {"msg":"034e186d-d8af-4bc6-8b56-c74fe923e02e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600032894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600055251Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600085177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600102254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600122522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600138959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600166416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600196782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600228197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600261782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:43091)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600294468Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:49220)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600327254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:36370)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.600361959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.601031117Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.601057304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.60122986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.601277144Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.601295151Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.601309379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.601622547Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.60174651Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602100771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602225344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602262989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602287106Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602374124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602393391Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602406629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602625219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602649786Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[0]R flush_numbers[0..12]: [258, 255, 258, 252, 252, 258, 249, 260, 258, 258, 249, 258]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602664014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[0]R generation[0..12]: [152, 150, 152, 150, 150, 152, 148, 153, 152, 152, 148, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602677622Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.6026912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[1]R flush_numbers[0..12]: [258, 255, 258, 252, 252, 258, 249, 260, 258, 258, 249, 258]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602704808Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[1]R generation[0..12]: [152, 150, 152, 150, 150, 152, 148, 153, 152, 152, 148, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602721336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602735064Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[2]R flush_numbers[0..12]: [258, 255, 258, 252, 252, 258, 249, 260, 258, 258, 249, 258]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602748522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[2]R generation[0..12]: [152, 150, 152, 150, 150, 152, 148, 153, 152, 152, 148, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602762351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602776049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602788627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602808025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Next flush: 261","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602820693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602833141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"":"downstairs","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602845809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602857448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"0","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602870116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"1","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602882664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"client":"2","":"downstairs","session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602895333Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602907201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"034e186d-d8af-4bc6-8b56-c74fe923e02e is now active with session: 6e085d5e-af25-4766-af6a-501422176e5c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602921829Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602933457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798,"session_id":"6e085d5e-af25-4766-af6a-501422176e5c"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:10:56.602945376Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 503 len 24576 data: 7 8 10 9 7 9 02/30 Flush 03/30 Write block 273 len 20480 data: 14 16 13 17 16 04/30 Write block 48 len 16384 data: 16 14 11 10 05/30 Write block 243 len 36864 data: 12 13 12 14 15 16 14 15 16 06/30 Write block 245 len 16384 data: 13 15 16 17 07/30 Read block 519 len 8192 08/30 Read block 143 len 28672 09/30 Read block 204 len 24576 10/30 Write block 417 len 28672 data: 14 13 12 12 13 11 15 11/30 Write block 160 len 4096 data: 13 12/30 Write block 78 len 32768 data: 9 11 10 10 10 11 12 9 13/30 Read block 351 len 24576 14/30 Write block 98 len 40960 data: 13 12 11 10 11 8 11 13 14 11 15/30 Read block 541 len 12288 16/30 Read block 97 len 32768 17/30 Read block 420 len 16384 18/30 Read block 228 len 20480 19/30 Write block 284 len 12288 data: 8 10 11 20/30 Flush 21/30 Write block 44 len 28672 data: 19 19 19 16 17 15 12 22/30 Read block 341 len 20480 23/30 Read block 261 len 36864 24/30 Flush 25/30 Write block 583 len 16384 data: 11 8 8 8 26/30 Read block 98 len 20480 27/30 Write block 374 len 28672 data: 12 14 13 15 18 16 16 28/30 Read block 252 len 36864 29/30 Write block 576 len 20480 data: 10 9 10 9 9 30/30 Write block 67 len 20480 data: 11 9 9 10 10 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:10:56.622399016Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 258 258 258 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 154 154 154 262 262 262 T T T 4 080-099 154 154 154 262 262 262 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 154 154 154 262 262 262 F F F 9 180-199 152 152 152 258 258 258 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 152 152 152 258 258 258 F F F 12 240-259 154 154 154 262 262 262 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 154 154 154 262 262 262 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 141 141 141 238 238 238 T T T 19 380-399 149 149 149 250 250 250 T T T 20 400-419 154 154 154 262 262 262 F F F 21 420-439 154 154 154 262 262 262 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 152 152 152 258 258 258 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 150 150 150 255 255 255 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 147 147 147 247 247 247 T T T 29 580-599 152 152 152 258 258 258 T T T Max gen: 154, Max flush: 263 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 258 258 258 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 155 155 155 264 264 264 F F F 4 080-099 154 154 154 262 262 262 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 148 148 148 249 249 249 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 154 154 154 262 262 262 F F F 9 180-199 152 152 152 258 258 258 F F F 10 200-219 148 148 148 249 249 249 F F F 11 220-239 152 152 152 258 258 258 F F F 12 240-259 154 154 154 262 262 262 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 154 154 154 262 262 262 F F F 15 300-319 146 146 146 246 246 246 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 154 154 154 262 262 262 F F F 21 420-439 154 154 154 262 262 262 F F F 22 440-459 152 152 152 259 259 259 F F F 23 460-479 152 152 152 258 258 258 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 150 150 150 255 255 255 F F F 27 540-559 150 150 150 256 256 256 F F F 28 560-579 155 155 155 264 264 264 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 155, Max flush: 264 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.897726652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.9438159Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.944543921Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.944586875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.944605192Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.94461981Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} {"msg":"Crucible 98fd7df6-2520-4530-bd84-c0258b52536f has session id: b43a95ae-ca84-451e-8f75-f2c1403f2c46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.945080277Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Upstairs opts: Upstairs UUID: 98fd7df6-2520-4530-bd84-c0258b52536f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.945125071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Crucible stats registered with UUID: 98fd7df6-2520-4530-bd84-c0258b52536f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.945149038Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.976906046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.97694669Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} {"msg":"98fd7df6-2520-4530-bd84-c0258b52536f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977009192Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977038628Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977065674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977081332Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.9770957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977108718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977122286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977139794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977154512Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.97716801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_connection connected from Ok(127.0.0.1:32824)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.97746113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_connection connected from Ok(127.0.0.1:41809)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977500984Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_connection connected from Ok(127.0.0.1:64176)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.977518042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978197699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978233454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978259661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978277448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978292326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978314783Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978882565Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.97892267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.978953886Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979414033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979442659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979457957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979554424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979576241Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979589729Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979602317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979616075Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[0]R flush_numbers[0..12]: [258, 255, 263, 264, 262, 262, 249, 260, 262, 258, 249, 258]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979629833Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[0]R generation[0..12]: [152, 150, 154, 155, 154, 154, 148, 153, 154, 152, 148, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979643651Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.97965743Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[1]R flush_numbers[0..12]: [258, 255, 263, 264, 262, 262, 249, 260, 262, 258, 249, 258]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979671198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[1]R generation[0..12]: [152, 150, 154, 155, 154, 154, 148, 153, 154, 152, 148, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979684726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979698404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[2]R flush_numbers[0..12]: [258, 255, 263, 264, 262, 262, 249, 260, 262, 258, 249, 258]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979711672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[2]R generation[0..12]: [152, 150, 154, 155, 154, 154, 148, 153, 154, 152, 148, 152]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.9797296Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979743638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979756146Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979775503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Next flush: 265","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979788142Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.97980069Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"":"downstairs","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979814058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979825816Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"0","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979838685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"1","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979851223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"client":"2","":"downstairs","session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979864131Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.9798761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"98fd7df6-2520-4530-bd84-c0258b52536f is now active with session: b43a95ae-ca84-451e-8f75-f2c1403f2c46","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979888778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:00.979900326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806,"session_id":"b43a95ae-ca84-451e-8f75-f2c1403f2c46"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:11:00.979912215Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 570 len 16384 data: 8 6 7 8 02/30 Write block 293 len 8192 data: 17 12 03/30 Read block 564 len 24576 04/30 Write block 294 len 36864 data: 13 13 14 17 18 18 16 13 12 05/30 Read block 138 len 28672 06/30 Read block 226 len 24576 07/30 Read block 447 len 16384 08/30 Write block 435 len 36864 data: 9 10 6 5 6 9 9 11 14 09/30 Read block 377 len 28672 10/30 Read block 283 len 40960 11/30 Write block 480 len 8192 data: 12 13 12/30 Read block 167 len 40960 13/30 Write block 68 len 32768 data: 10 10 11 11 10 10 10 8 14/30 Read block 564 len 20480 15/30 Write block 460 len 28672 data: 9 10 12 12 10 9 8 16/30 Write block 9 len 12288 data: 12 13 13 17/30 Read block 143 len 32768 18/30 Read block 474 len 20480 19/30 Write block 60 len 4096 data: 18 20/30 Write block 244 len 20480 data: 14 14 16 17 18 21/30 Flush 22/30 Write block 520 len 16384 data: 9 10 12 13 23/30 Read block 587 len 32768 24/30 Write block 547 len 32768 data: 14 12 11 11 12 13 12 15 25/30 Write block 121 len 28672 data: 10 11 8 8 8 11 13 26/30 Read block 274 len 36864 27/30 Read block 453 len 8192 28/30 Read block 314 len 40960 29/30 Write block 214 len 40960 data: 11 10 14 15 14 14 13 11 13 12 30/30 Write block 176 len 36864 data: 14 14 13 11 12 12 14 13 12 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 4 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:01.000549733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 156 156 156 265 265 265 F F F 4 080-099 154 154 154 262 262 262 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 148 148 148 249 249 249 T T T 7 140-159 153 153 153 260 260 260 F F F 8 160-179 154 154 154 262 262 262 T T T 9 180-199 152 152 152 258 258 258 T T T 10 200-219 148 148 148 249 249 249 T T T 11 220-239 152 152 152 258 258 258 T T T 12 240-259 156 156 156 265 265 265 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 154 154 154 262 262 262 F F F 21 420-439 156 156 156 265 265 265 F F F 22 440-459 156 156 156 265 265 265 F F F 23 460-479 156 156 156 265 265 265 F F F 24 480-499 156 156 156 265 265 265 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 150 150 150 255 255 255 T T T 27 540-559 150 150 150 256 256 256 T T T 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 156, Max flush: 265 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 156 156 156 265 265 265 F F F 4 080-099 154 154 154 262 262 262 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 157 157 157 266 266 266 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 157 157 157 266 266 266 F F F 9 180-199 157 157 157 266 266 266 F F F 10 200-219 157 157 157 266 266 266 F F F 11 220-239 157 157 157 266 266 266 F F F 12 240-259 156 156 156 265 265 265 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 148 148 148 249 249 249 F F F 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 154 154 154 262 262 262 F F F 21 420-439 156 156 156 265 265 265 F F F 22 440-459 156 156 156 265 265 265 F F F 23 460-479 156 156 156 265 265 265 F F F 24 480-499 156 156 156 265 265 265 F F F 25 500-519 154 154 154 261 261 261 F F F 26 520-539 157 157 157 266 266 266 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 157, Max flush: 266 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.285981049Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:05.330412284Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.330791592Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.363573859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.364280043Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.364315048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.364333026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.364347704Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} {"msg":"Crucible a8c4a337-795f-4a79-8323-f9e3d0dcbe2b has session id: b2a5688b-a032-46f9-836b-e9ccbcb5dcc2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.36496122Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Upstairs opts: Upstairs UUID: a8c4a337-795f-4a79-8323-f9e3d0dcbe2b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.365026161Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Crucible stats registered with UUID: a8c4a337-795f-4a79-8323-f9e3d0dcbe2b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.365053307Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.396795397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.396829222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} {"msg":"a8c4a337-795f-4a79-8323-f9e3d0dcbe2b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.396889714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.39691641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.396940817Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.396957095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.396975742Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397000429Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397026885Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397051992Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397081078Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397107254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50698)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397130941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50001)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397163057Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59633)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397373618Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397919613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397947009Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397982345Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.397997892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.39801121Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.398024759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.398702636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.398731112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.39874694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399270548Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399301484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399316442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399389252Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399417948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399432446Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399445034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399458363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[0]R flush_numbers[0..12]: [265, 255, 263, 265, 262, 262, 266, 260, 266, 266, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399471871Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[0]R generation[0..12]: [156, 150, 154, 156, 154, 154, 157, 153, 157, 157, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399485329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399499117Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[1]R flush_numbers[0..12]: [265, 255, 263, 265, 262, 262, 266, 260, 266, 266, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399512525Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[1]R generation[0..12]: [156, 150, 154, 156, 154, 154, 157, 153, 157, 157, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399525943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399539652Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[2]R flush_numbers[0..12]: [265, 255, 263, 265, 262, 262, 266, 260, 266, 266, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.39955298Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[2]R generation[0..12]: [156, 150, 154, 156, 154, 154, 157, 153, 157, 157, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399566438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399583536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399596564Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399609422Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.39962784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399640628Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"":"downstairs","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399653636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399665454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"0","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399678133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"1","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399690901Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"client":"2","":"downstairs","session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399703749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399715748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"a8c4a337-795f-4a79-8323-f9e3d0dcbe2b is now active with session: b2a5688b-a032-46f9-836b-e9ccbcb5dcc2","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399728266Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399739894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814,"session_id":"b2a5688b-a032-46f9-836b-e9ccbcb5dcc2"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:05.399751843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 174 len 12288 data: 16 15 15 02/30 Read block 145 len 16384 03/30 Write block 419 len 28672 data: 13 13 14 12 16 13 14 04/30 Write block 502 len 12288 data: 7 8 9 05/30 Flush 06/30 Write block 197 len 12288 data: 12 11 11 07/30 Read block 496 len 12288 08/30 Write block 401 len 4096 data: 9 09/30 Read block 402 len 32768 10/30 Read block 294 len 16384 11/30 Read block 25 len 32768 12/30 Write block 496 len 32768 data: 14 15 13 12 11 10 8 9 13/30 Read block 85 len 4096 14/30 Write block 71 len 12288 data: 12 11 11 15/30 Write block 488 len 16384 data: 15 12 15 15 16/30 Read block 282 len 8192 17/30 Write block 349 len 8192 data: 12 9 18/30 Write block 81 len 8192 data: 11 11 19/30 Read block 115 len 36864 20/30 Read block 277 len 16384 21/30 Read block 9 len 16384 22/30 Flush 23/30 Write block 407 len 16384 data: 15 15 15 14 24/30 Flush 25/30 Write block 178 len 28672 data: 14 12 13 13 15 14 13 26/30 Read block 197 len 36864 27/30 Read block 231 len 40960 28/30 Read block 256 len 20480 29/30 Write block 244 len 20480 data: 15 15 17 18 19 30/30 Write block 409 len 12288 data: 16 15 15 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:05.41834761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 158 158 158 268 268 268 F F F 4 080-099 158 158 158 268 268 268 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 157 157 157 266 266 266 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 158 158 158 267 267 267 T T T 9 180-199 158 158 158 268 268 268 T T T 10 200-219 157 157 157 266 266 266 F F F 11 220-239 157 157 157 266 266 266 F F F 12 240-259 156 156 156 265 265 265 T T T 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 158 158 158 268 268 268 F F F 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 158 158 158 269 269 269 T T T 21 420-439 158 158 158 267 267 267 F F F 22 440-459 156 156 156 265 265 265 F F F 23 460-479 156 156 156 265 265 265 F F F 24 480-499 158 158 158 268 268 268 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 157 157 157 266 266 266 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 158, Max flush: 269 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 F F F 1 020-039 150 150 150 255 255 255 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 158 158 158 268 268 268 F F F 4 080-099 158 158 158 268 268 268 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 157 157 157 266 266 266 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 159 159 159 270 270 270 F F F 10 200-219 157 157 157 266 266 266 F F F 11 220-239 157 157 157 266 266 266 F F F 12 240-259 159 159 159 270 270 270 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 144 144 144 243 243 243 F F F 17 340-359 158 158 158 268 268 268 F F F 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 159 159 159 270 270 270 F F F 21 420-439 158 158 158 267 267 267 F F F 22 440-459 156 156 156 265 265 265 F F F 23 460-479 156 156 156 265 265 265 F F F 24 480-499 158 158 158 268 268 268 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 157 157 157 266 266 266 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 159, Max flush: 270 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.720996241Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:09.764552365Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.764980987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.797066549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.79764951Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.797697283Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.797714541Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.797732098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} {"msg":"Crucible 0974276a-94b0-4402-83a0-3670b8501003 has session id: 609ad22d-ec82-4c26-b601-2c473a9a8036","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.798280533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Upstairs opts: Upstairs UUID: 0974276a-94b0-4402-83a0-3670b8501003, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.798324487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Crucible stats registered with UUID: 0974276a-94b0-4402-83a0-3670b8501003","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.798348404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829301452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829332937Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} {"msg":"0974276a-94b0-4402-83a0-3670b8501003 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829401488Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829425325Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829453771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829471938Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829488766Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829505324Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829521932Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829548708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829580393Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829611919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_connection connected from Ok(127.0.0.1:54757)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829649354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_connection connected from Ok(127.0.0.1:34694)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829848637Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_connection connected from Ok(127.0.0.1:34090)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.829878943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.83048564Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.830515946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.830541502Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.830568569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.830608803Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.83063043Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831094326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831132521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831225629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831618505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831644571Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831658799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831688025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831702553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831715432Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831781453Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.83180004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[0]R flush_numbers[0..12]: [265, 255, 263, 268, 268, 262, 266, 260, 270, 270, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831814058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[0]R generation[0..12]: [156, 150, 154, 158, 158, 154, 157, 153, 159, 159, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831827846Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831841534Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[1]R flush_numbers[0..12]: [265, 255, 263, 268, 268, 262, 266, 260, 270, 270, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831854922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[1]R generation[0..12]: [156, 150, 154, 158, 158, 154, 157, 153, 159, 159, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831868391Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831881979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[2]R flush_numbers[0..12]: [265, 255, 263, 268, 268, 262, 266, 260, 270, 270, 266, 266]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831898987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[2]R generation[0..12]: [156, 150, 154, 158, 158, 154, 157, 153, 159, 159, 157, 157]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831923363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831951669Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.831973726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:11:09.831998083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832027219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832052675Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"":"downstairs","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832068713Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832080722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"0","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.83209407Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"1","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832116177Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"client":"2","":"downstairs","session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832141114Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.83216524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"0974276a-94b0-4402-83a0-3670b8501003 is now active with session: 609ad22d-ec82-4c26-b601-2c473a9a8036","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832182528Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832195226Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822,"session_id":"609ad22d-ec82-4c26-b601-2c473a9a8036"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:09.832216353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 330 len 16384 data: 6 7 10 12 02/30 Flush 03/30 Write block 425 len 24576 data: 15 11 12 13 13 16 04/30 Flush 05/30 Write block 90 len 4096 data: 13 06/30 Flush 07/30 Write block 454 len 32768 data: 11 10 10 11 12 8 10 11 08/30 Write block 27 len 16384 data: 9 13 11 10 09/30 Write block 483 len 12288 data: 12 10 11 10/30 Read block 246 len 32768 11/30 Write block 442 len 40960 data: 12 15 16 18 17 16 16 17 16 13 12/30 Write block 432 len 36864 data: 10 9 8 10 11 7 6 7 10 13/30 Write block 134 len 12288 data: 16 17 13 14/30 Write block 355 len 20480 data: 13 14 13 16 14 15/30 Read block 298 len 8192 16/30 Read block 278 len 28672 17/30 Read block 165 len 8192 18/30 Write block 89 len 20480 data: 11 14 12 13 12 19/30 Write block 187 len 4096 data: 10 20/30 Write block 436 len 36864 data: 12 8 7 8 11 10 13 16 17 21/30 Read block 415 len 28672 22/30 Write block 222 len 36864 data: 14 13 12 12 11 7 9 10 11 23/30 Write block 232 len 40960 data: 11 11 14 14 14 16 15 14 13 14 24/30 Read block 569 len 20480 25/30 Read block 581 len 24576 26/30 Write block 527 len 12288 data: 15 15 13 27/30 Read block 327 len 4096 28/30 Write block 195 len 36864 data: 8 7 13 12 12 11 10 9 9 29/30 Read block 304 len 20480 30/30 Write block 487 len 40960 data: 12 16 13 16 16 15 17 16 16 15 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 8 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 3 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Write 9 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:09.851137386Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 F F F 1 020-039 150 150 150 255 255 255 T T T 2 040-059 154 154 154 263 263 263 F F F 3 060-079 158 158 158 268 268 268 F F F 4 080-099 160 160 160 273 273 273 T T T 5 100-119 154 154 154 262 262 262 F F F 6 120-139 157 157 157 266 266 266 T T T 7 140-159 153 153 153 260 260 260 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 159 159 159 270 270 270 T T T 10 200-219 157 157 157 266 266 266 T T T 11 220-239 157 157 157 266 266 266 T T T 12 240-259 159 159 159 270 270 270 T T T 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 160 160 160 271 271 271 F F F 17 340-359 158 158 158 268 268 268 T T T 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 159 159 159 270 270 270 F F F 21 420-439 160 160 160 272 272 272 T T T 22 440-459 156 156 156 265 265 265 T T T 23 460-479 156 156 156 265 265 265 T T T 24 480-499 158 158 158 268 268 268 T T T 25 500-519 158 158 158 268 268 268 F F F 26 520-539 157 157 157 266 266 266 T T T 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 160, Max flush: 273 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 F F F 1 020-039 161 161 161 274 274 274 F F F 2 040-059 154 154 154 263 263 263 F F F 3 060-079 158 158 158 268 268 268 F F F 4 080-099 161 161 161 274 274 274 F F F 5 100-119 154 154 154 262 262 262 F F F 6 120-139 161 161 161 274 274 274 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 274 274 274 F F F 10 200-219 161 161 161 274 274 274 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 160 160 160 271 271 271 F F F 17 340-359 161 161 161 274 274 274 F F F 18 360-379 155 155 155 264 264 264 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 159 159 159 270 270 270 F F F 21 420-439 161 161 161 274 274 274 F F F 22 440-459 161 161 161 274 274 274 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 161 161 161 274 274 274 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 161 161 161 274 274 274 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 161, Max flush: 274 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.210212313Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.255225578Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.255876339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.25594072Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.255962247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.255980125Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} {"msg":"Crucible 8e156c02-baca-4185-a8d5-e056eda2dcbb has session id: 48d9dbe6-347e-4e7d-84e2-7a3bc97667e8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.256470158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Upstairs opts: Upstairs UUID: 8e156c02-baca-4185-a8d5-e056eda2dcbb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.256574223Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Crucible stats registered with UUID: 8e156c02-baca-4185-a8d5-e056eda2dcbb","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.256608869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.287803433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.287844738Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} {"msg":"8e156c02-baca-4185-a8d5-e056eda2dcbb active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.287923777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.287957772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.287989658Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288016794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.28804638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288068487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288093964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.28812476Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288154565Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288178522Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:54494)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288400442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:40437)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288434737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_connection connected from Ok(127.0.0.1:43083)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.288450645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289045404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.28907218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289092657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289106425Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289119553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289133942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.28972667Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289754387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.289769585Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290298322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290336527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290365723Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.29038763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290410907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290432584Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290451451Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290473338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[0]R flush_numbers[0..12]: [265, 274, 263, 268, 274, 262, 274, 260, 270, 274, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290498145Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[0]R generation[0..12]: [156, 161, 154, 158, 161, 154, 161, 153, 159, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290524381Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290549158Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[1]R flush_numbers[0..12]: [265, 274, 263, 268, 274, 262, 274, 260, 270, 274, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290574654Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[1]R generation[0..12]: [156, 161, 154, 158, 161, 154, 161, 153, 159, 161, 161, 161]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:11:14.290601761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290629447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[2]R flush_numbers[0..12]: [265, 274, 263, 268, 274, 262, 274, 260, 270, 274, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290655754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[2]R generation[0..12]: [156, 161, 154, 158, 161, 154, 161, 153, 159, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.29068304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290714486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290741962Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Generation requested: 162 >= found:162"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290771638Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290797504Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.29082318Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"":"downstairs","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290843028Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290855966Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"0","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290869444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"1","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290882352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"client":"2","":"downstairs","session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290895741Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290907659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"8e156c02-baca-4185-a8d5-e056eda2dcbb is now active with session: 48d9dbe6-347e-4e7d-84e2-7a3bc97667e8","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290920217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290931806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830,"session_id":"48d9dbe6-347e-4e7d-84e2-7a3bc97667e8"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:14.290943674Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 24576 data: 13 11 12 13 13 17 02/30 Read block 585 len 28672 03/30 Write block 425 len 16384 data: 16 12 13 14 04/30 Read block 247 len 8192 05/30 Write block 486 len 20480 data: 14 14 18 14 17 06/30 Read block 57 len 40960 07/30 Write block 430 len 8192 data: 17 11 08/30 Flush 09/30 Write block 60 len 32768 data: 19 17 14 13 12 9 9 12 10/30 Read block 373 len 24576 11/30 Read block 571 len 8192 12/30 Write block 50 len 16384 data: 13 11 9 9 13/30 Read block 140 len 32768 14/30 Read block 511 len 4096 15/30 Read block 21 len 28672 16/30 Read block 504 len 20480 17/30 Write block 70 len 28672 data: 12 13 12 12 11 9 9 18/30 Read block 589 len 32768 19/30 Write block 431 len 36864 data: 12 11 10 9 11 13 9 8 9 20/30 Write block 447 len 12288 data: 17 17 18 21/30 Read block 44 len 28672 22/30 Write block 11 len 20480 data: 14 12 13 17 19 23/30 Write block 369 len 36864 data: 14 13 15 13 11 13 15 14 16 24/30 Write block 92 len 8192 data: 14 13 25/30 Read block 283 len 8192 26/30 Read block 298 len 36864 27/30 Read block 299 len 12288 28/30 Read block 130 len 4096 29/30 Write block 113 len 4096 data: 11 30/30 Read block 551 len 12288 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 8 Done Done Done false 1009 Acked Read 6 Done Done Done false 1010 Acked Read 2 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 1 Sent Done Done false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 1 3 Done 20 22 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:14.310182344Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 156 156 156 265 265 265 T T T 1 020-039 161 161 161 274 274 274 F F F 2 040-059 154 154 154 263 263 263 T T T 3 060-079 158 158 158 268 268 268 T T T 4 080-099 161 161 161 274 274 274 T T T 5 100-119 154 154 154 262 262 262 T T T 6 120-139 161 161 161 274 274 274 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 274 274 274 F F F 10 200-219 161 161 161 274 274 274 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 160 160 160 271 271 271 F F F 17 340-359 161 161 161 274 274 274 F F F 18 360-379 155 155 155 264 264 264 T T T 19 380-399 155 155 155 264 264 264 F F F 20 400-419 159 159 159 270 270 270 F F F 21 420-439 162 162 162 275 275 275 T T T 22 440-459 161 161 161 274 274 274 T T T 23 460-479 161 161 161 274 274 274 F F F 24 480-499 162 162 162 275 275 275 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 161 161 161 274 274 274 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 162, Max flush: 275 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 163 163 163 276 276 276 F F F 1 020-039 161 161 161 274 274 274 F F F 2 040-059 163 163 163 276 276 276 F F F 3 060-079 163 163 163 276 276 276 F F F 4 080-099 163 163 163 276 276 276 F F F 5 100-119 163 163 163 276 276 276 F F F 6 120-139 161 161 161 274 274 274 F F F 7 140-159 153 153 153 260 260 260 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 274 274 274 F F F 10 200-219 161 161 161 274 274 274 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 F F F 16 320-339 160 160 160 271 271 271 F F F 17 340-359 161 161 161 274 274 274 F F F 18 360-379 163 163 163 276 276 276 F F F 19 380-399 155 155 155 264 264 264 F F F 20 400-419 159 159 159 270 270 270 F F F 21 420-439 163 163 163 276 276 276 F F F 22 440-459 163 163 163 276 276 276 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 162 162 162 275 275 275 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 161 161 161 274 274 274 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 163, Max flush: 276 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.604971859Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.651743424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.652371228Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.652413663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.65242889Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.652446008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} {"msg":"Crucible cf3aebc3-9d39-4618-aba2-f97f4f0769cf has session id: f379235a-1522-44d4-88ea-ffbc18713bf9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.653027169Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Upstairs opts: Upstairs UUID: cf3aebc3-9d39-4618-aba2-f97f4f0769cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.653061824Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Crucible stats registered with UUID: cf3aebc3-9d39-4618-aba2-f97f4f0769cf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.653078572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684542379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684574475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} {"msg":"cf3aebc3-9d39-4618-aba2-f97f4f0769cf active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684645765Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684667802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684706387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684739433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684771978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684805963Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684837579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684865605Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684892772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684919278Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55447)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.684951414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:51215)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.68498034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_connection connected from Ok(127.0.0.1:36484)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.685007616Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.685624312Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.685659007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.685686323Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.68571217Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.685737336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.685762433Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.68621749Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.68629359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686372879Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686762526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686789552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.68680424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686826487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686840765Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686853554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686916565Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686934943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[0]R flush_numbers[0..12]: [276, 274, 276, 276, 276, 276, 274, 260, 270, 274, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686948781Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[0]R generation[0..12]: [163, 161, 163, 163, 163, 163, 161, 153, 159, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686962529Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686976307Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[1]R flush_numbers[0..12]: [276, 274, 276, 276, 276, 276, 274, 260, 270, 274, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.686989675Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[1]R generation[0..12]: [163, 161, 163, 163, 163, 163, 161, 153, 159, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687002813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687016311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[2]R flush_numbers[0..12]: [276, 274, 276, 276, 276, 276, 274, 260, 270, 274, 274, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687029539Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[2]R generation[0..12]: [163, 161, 163, 163, 163, 163, 161, 153, 159, 161, 161, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687042738Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687056286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687071374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687084752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.68709724Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:11:18.687109819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"":"downstairs","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687127566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687139155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"0","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687152053Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"1","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687164931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"client":"2","":"downstairs","session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687177989Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687190008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"cf3aebc3-9d39-4618-aba2-f97f4f0769cf is now active with session: f379235a-1522-44d4-88ea-ffbc18713bf9","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687202486Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687214144Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838,"session_id":"f379235a-1522-44d4-88ea-ffbc18713bf9"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:18.687225973Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 32768 data: 8 11 13 10 11 14 13 15 02/30 Read block 284 len 4096 03/30 Write block 87 len 24576 data: 8 9 12 15 13 15 04/30 Read block 519 len 20480 05/30 Flush 06/30 Write block 145 len 12288 data: 16 14 14 07/30 Write block 383 len 12288 data: 13 16 18 08/30 Read block 295 len 16384 09/30 Write block 19 len 40960 data: 16 14 12 11 10 9 8 8 10 14 10/30 Flush 11/30 Write block 211 len 20480 data: 11 11 14 12 11 12/30 Read block 567 len 8192 13/30 Read block 489 len 28672 14/30 Flush 15/30 Write block 18 len 28672 data: 15 17 15 13 12 11 10 16/30 Write block 148 len 12288 data: 14 12 12 17/30 Write block 445 len 20480 data: 19 18 18 18 19 18/30 Write block 525 len 4096 data: 13 19/30 Write block 65 len 36864 data: 10 10 13 11 11 13 14 13 13 20/30 Read block 170 len 24576 21/30 Read block 542 len 24576 22/30 Flush 23/30 Write block 408 len 32768 data: 16 17 16 16 18 14 14 15 24/30 Read block 433 len 28672 25/30 Read block 365 len 16384 26/30 Read block 542 len 8192 27/30 Read block 285 len 24576 28/30 Write block 314 len 16384 data: 15 14 17 19 29/30 Read block 448 len 24576 30/30 Write block 35 len 40960 data: 12 12 12 12 13 13 18 20 19 20 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 8 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:18.707223338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 F F F 1 020-039 164 164 164 280 280 280 T T T 2 040-059 163 163 163 276 276 276 T T T 3 060-079 164 164 164 280 280 280 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 163 163 163 276 276 276 F F F 6 120-139 161 161 161 274 274 274 F F F 7 140-159 164 164 164 280 280 280 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 274 274 274 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 156 156 156 265 265 265 T T T 16 320-339 164 164 164 277 277 277 F F F 17 340-359 161 161 161 274 274 274 F F F 18 360-379 163 163 163 276 276 276 F F F 19 380-399 164 164 164 278 278 278 F F F 20 400-419 159 159 159 270 270 270 T T T 21 420-439 163 163 163 276 276 276 F F F 22 440-459 164 164 164 280 280 280 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 162 162 162 275 275 275 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 164, Max flush: 280 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 F F F 1 020-039 165 165 165 281 281 281 F F F 2 040-059 165 165 165 281 281 281 F F F 3 060-079 164 164 164 280 280 280 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 163 163 163 276 276 276 F F F 6 120-139 161 161 161 274 274 274 F F F 7 140-159 164 164 164 280 280 280 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 161 161 161 274 274 274 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 154 154 154 262 262 262 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 165 165 165 281 281 281 F F F 16 320-339 164 164 164 277 277 277 F F F 17 340-359 161 161 161 274 274 274 F F F 18 360-379 163 163 163 276 276 276 F F F 19 380-399 164 164 164 278 278 278 F F F 20 400-419 165 165 165 281 281 281 F F F 21 420-439 163 163 163 276 276 276 F F F 22 440-459 164 164 164 280 280 280 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 162 162 162 275 275 275 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 157 157 157 266 266 266 F F F 28 560-579 156 156 156 265 265 265 F F F 29 580-599 155 155 155 264 264 264 F F F Max gen: 165, Max flush: 281 Begin loop 83 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:22.982695136Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.0288174Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.029493437Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.029536201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.029551359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.029565717Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} {"msg":"Crucible 82de32b3-23b9-463e-8fed-e516168274b1 has session id: d442f759-134f-4536-8e8c-0f12aefbe019","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.030248484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Upstairs opts: Upstairs UUID: 82de32b3-23b9-463e-8fed-e516168274b1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.030296208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Crucible stats registered with UUID: 82de32b3-23b9-463e-8fed-e516168274b1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.030324244Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061400884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.06143426Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} {"msg":"82de32b3-23b9-463e-8fed-e516168274b1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061517588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061540736Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061568712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061596098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061627544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061679406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061714772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061747857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061779643Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061814348Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_connection connected from Ok(127.0.0.1:50349)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061868751Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_connection connected from Ok(127.0.0.1:54958)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.061939611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_connection connected from Ok(127.0.0.1:34637)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.062034668Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.062638915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.06267748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.062705936Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.062732593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.062760059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.062789355Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063298905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063431387Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063467582Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063890814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063914801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063928329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.063987311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064006718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064019977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064083978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064102735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[0]R flush_numbers[0..12]: [280, 281, 281, 280, 277, 276, 274, 280, 270, 274, 279, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064116523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[0]R generation[0..12]: [164, 165, 165, 164, 164, 163, 161, 164, 159, 161, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064130061Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064143589Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[1]R flush_numbers[0..12]: [280, 281, 281, 280, 277, 276, 274, 280, 270, 274, 279, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064157208Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[1]R generation[0..12]: [164, 165, 165, 164, 164, 163, 161, 164, 159, 161, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064170586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064184224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[2]R flush_numbers[0..12]: [280, 281, 281, 280, 277, 276, 274, 280, 270, 274, 279, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064197662Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[2]R generation[0..12]: [164, 165, 165, 164, 164, 163, 161, 164, 159, 161, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.06421112Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064225008Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064237477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064250275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064262883Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064278021Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","Wait for a query_work_queue command to finish before sending IO pid":1846,"":"downstairs","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064297928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064309657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"0","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064322685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"1","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064335613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"client":"2","":"downstairs","session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064348841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.06436097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"82de32b3-23b9-463e-8fed-e516168274b1 is now active with session: d442f759-134f-4536-8e8c-0f12aefbe019","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064373478Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064385496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846,"session_id":"d442f759-134f-4536-8e8c-0f12aefbe019"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:23.064397395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 451 len 8192 data: 14 15 02/30 Write block 128 len 4096 data: 17 03/30 Write block 345 len 40960 data: 15 11 12 13 13 10 9 9 12 11 04/30 Read block 226 len 28672 05/30 Write block 71 len 32768 data: 15 14 14 12 10 10 8 10 06/30 Read block 160 len 28672 07/30 Read block 201 len 40960 08/30 Write block 344 len 4096 data: 14 09/30 Write block 274 len 24576 data: 17 14 18 17 14 11 10/30 Write block 564 len 24576 data: 9 11 10 10 10 10 11/30 Write block 303 len 36864 data: 10 10 7 9 8 9 8 10 10 12/30 Write block 329 len 12288 data: 7 7 9 13/30 Read block 208 len 24576 14/30 Write block 182 len 12288 data: 16 15 14 15/30 Write block 417 len 24576 data: 15 14 14 14 15 13 16/30 Read block 391 len 24576 17/30 Write block 154 len 16384 data: 10 14 15 14 18/30 Write block 482 len 40960 data: 12 14 12 13 15 15 19 15 18 17 19/30 Flush 20/30 Write block 34 len 16384 data: 12 13 13 13 21/30 Read block 117 len 40960 22/30 Read block 189 len 40960 23/30 Read block 555 len 36864 24/30 Write block 576 len 12288 data: 11 10 11 25/30 Write block 581 len 16384 data: 10 14 12 9 26/30 Write block 142 len 36864 data: 13 14 15 17 15 15 15 13 13 27/30 Read block 176 len 28672 28/30 Read block 41 len 8192 29/30 Read block 544 len 28672 30/30 Write block 552 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Write 4 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:23.084068515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 F F F 1 020-039 165 165 165 281 281 281 T T T 2 040-059 165 165 165 281 281 281 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 163 163 163 276 276 276 F F F 6 120-139 166 166 166 282 282 282 F F F 7 140-159 166 166 166 282 282 282 T T T 8 160-179 159 159 159 270 270 270 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 166 166 166 282 282 282 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 163 163 163 276 276 276 F F F 19 380-399 164 164 164 278 278 278 F F F 20 400-419 166 166 166 282 282 282 F F F 21 420-439 166 166 166 282 282 282 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 166 166 166 282 282 282 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 157 157 157 266 266 266 T T T 28 560-579 166 166 166 282 282 282 T T T 29 580-599 155 155 155 264 264 264 T T T Max gen: 166, Max flush: 282 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 F F F 1 020-039 167 167 167 283 283 283 F F F 2 040-059 165 165 165 281 281 281 F F F 3 060-079 166 166 166 282 282 282 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 163 163 163 276 276 276 F F F 6 120-139 166 166 166 282 282 282 F F F 7 140-159 167 167 167 283 283 283 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 166 166 166 282 282 282 F F F 10 200-219 164 164 164 279 279 279 F F F 11 220-239 161 161 161 274 274 274 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 166 166 166 282 282 282 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 163 163 163 276 276 276 F F F 19 380-399 164 164 164 278 278 278 F F F 20 400-419 166 166 166 282 282 282 F F F 21 420-439 166 166 166 282 282 282 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 166 166 166 282 282 282 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 167 167 167 283 283 283 F F F 28 560-579 167 167 167 283 283 283 F F F 29 580-599 167 167 167 283 283 283 F F F Max gen: 167, Max flush: 283 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.365612741Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:27.409415002Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.40980143Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.441704447Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.442353229Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.442387364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.442400922Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.4424151Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} {"msg":"Crucible 78a6a675-c6d4-429e-a776-f5d96ab540bc has session id: 3c1094c0-ecbc-41a9-b6c7-4afd2bb549de","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.443084949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Upstairs opts: Upstairs UUID: 78a6a675-c6d4-429e-a776-f5d96ab540bc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.443117444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Crucible stats registered with UUID: 78a6a675-c6d4-429e-a776-f5d96ab540bc","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.443135372Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475078594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475117789Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} {"msg":"78a6a675-c6d4-429e-a776-f5d96ab540bc active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475185139Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475218395Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475250251Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475268138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475285096Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475301304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475317481Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475333349Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475351367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475367905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_connection connected from Ok(127.0.0.1:50848)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475392861Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_connection connected from Ok(127.0.0.1:51373)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475445034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_connection connected from Ok(127.0.0.1:46790)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.475482329Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476113353Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476142509Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476158406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476179314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476330013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476371197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476868729Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476899245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.476926831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477419124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.47744595Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477460348Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477538708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477558645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477572013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477587651Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477600759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[0]R flush_numbers[0..12]: [280, 283, 281, 282, 277, 276, 282, 283, 270, 282, 279, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477614907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[0]R generation[0..12]: [164, 167, 165, 166, 164, 163, 166, 167, 159, 166, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477628716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477642523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[1]R flush_numbers[0..12]: [280, 283, 281, 282, 277, 276, 282, 283, 270, 282, 279, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477656162Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[1]R generation[0..12]: [164, 167, 165, 166, 164, 163, 166, 167, 159, 166, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.47766949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477683188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[2]R flush_numbers[0..12]: [280, 283, 281, 282, 277, 276, 282, 283, 270, 282, 279, 274]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477697306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[2]R generation[0..12]: [164, 167, 165, 166, 164, 163, 166, 167, 159, 166, 164, 161]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477710594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477724272Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477736841Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477749579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Next flush: 284","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477762137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477776335Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"":"downstairs","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477789394Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477819109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"0","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477833697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"1","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477846706Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"client":"2","":"downstairs","session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477859774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477871732Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"78a6a675-c6d4-429e-a776-f5d96ab540bc is now active with session: 3c1094c0-ecbc-41a9-b6c7-4afd2bb549de","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.47788419Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477895909Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854,"session_id":"3c1094c0-ecbc-41a9-b6c7-4afd2bb549de"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:27.477907767Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 372 len 32768 data: 14 12 14 16 15 17 19 17 02/30 Write block 40 len 8192 data: 14 19 03/30 Read block 471 len 8192 04/30 Read block 521 len 12288 05/30 Write block 323 len 16384 data: 13 15 11 9 06/30 Read block 371 len 4096 07/30 Read block 243 len 28672 08/30 Read block 429 len 40960 09/30 Read block 448 len 40960 10/30 Read block 493 len 12288 11/30 Read block 75 len 4096 12/30 Write block 197 len 24576 data: 14 13 13 12 11 10 13/30 Read block 578 len 36864 14/30 Read block 250 len 24576 15/30 Read block 313 len 20480 16/30 Read block 550 len 12288 17/30 Read block 539 len 24576 18/30 Read block 193 len 16384 19/30 Write block 368 len 4096 data: 13 20/30 Flush 21/30 Write block 68 len 40960 data: 12 12 14 16 15 15 13 11 11 9 22/30 Write block 331 len 28672 data: 10 12 14 11 12 15 14 23/30 Write block 107 len 28672 data: 12 11 10 10 12 11 12 24/30 Read block 4 len 20480 25/30 Read block 230 len 24576 26/30 Write block 225 len 8192 data: 13 12 27/30 Read block 112 len 16384 28/30 Write block 374 len 24576 data: 15 17 16 18 20 18 29/30 Read block 473 len 4096 30/30 Read block 505 len 20480 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 10 Done Done Done false 1021 Acked Write 7 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 5 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:27.497720248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 F F F 1 020-039 167 167 167 283 283 283 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 166 166 166 282 282 282 T T T 4 080-099 164 164 164 277 277 277 F F F 5 100-119 163 163 163 276 276 276 T T T 6 120-139 166 166 166 282 282 282 F F F 7 140-159 167 167 167 283 283 283 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 168 168 168 284 284 284 F F F 10 200-219 168 168 168 284 284 284 F F F 11 220-239 161 161 161 274 274 274 T T T 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 168 168 168 284 284 284 T T T 17 340-359 166 166 166 282 282 282 F F F 18 360-379 168 168 168 284 284 284 T T T 19 380-399 164 164 164 278 278 278 F F F 20 400-419 166 166 166 282 282 282 F F F 21 420-439 166 166 166 282 282 282 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 166 166 166 282 282 282 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 167 167 167 283 283 283 F F F 28 560-579 167 167 167 283 283 283 F F F 29 580-599 167 167 167 283 283 283 F F F Max gen: 168, Max flush: 284 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 F F F 1 020-039 167 167 167 283 283 283 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 169 169 169 285 285 285 F F F 6 120-139 166 166 166 282 282 282 F F F 7 140-159 167 167 167 283 283 283 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 168 168 168 284 284 284 F F F 10 200-219 168 168 168 284 284 284 F F F 11 220-239 169 169 169 285 285 285 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 169 169 169 285 285 285 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 169 169 169 285 285 285 F F F 19 380-399 164 164 164 278 278 278 F F F 20 400-419 166 166 166 282 282 282 F F F 21 420-439 166 166 166 282 282 282 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 161 161 161 274 274 274 F F F 24 480-499 166 166 166 282 282 282 F F F 25 500-519 158 158 158 268 268 268 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 167 167 167 283 283 283 F F F 28 560-579 167 167 167 283 283 283 F F F 29 580-599 167 167 167 283 283 283 F F F Max gen: 169, Max flush: 285 Begin loop 85 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.810437593Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.856036778Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.856695658Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.856736262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.856750211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.856764849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} {"msg":"Crucible 76044c40-4a64-4a9e-9ab9-9aa4bb704cae has session id: 070b798c-eb28-435d-bf91-e7bc4d43ce5b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.857305764Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Upstairs opts: Upstairs UUID: 76044c40-4a64-4a9e-9ab9-9aa4bb704cae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.857384254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Crucible stats registered with UUID: 76044c40-4a64-4a9e-9ab9-9aa4bb704cae","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.85741608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889422713Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.88944878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} {"msg":"76044c40-4a64-4a9e-9ab9-9aa4bb704cae active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889530738Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889554405Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889583371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889601899Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889618606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889634514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.88966218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889689957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889719392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889748988Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:48185)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889780274Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:45272)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.88981198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36880)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.889845085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.890586424Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.89061311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.890627838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.890644836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.890658304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.890674752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891179903Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891320184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891466414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891802248Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891826024Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891842322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891945428Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891972384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.891987173Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892012929Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892026837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[0]R flush_numbers[0..12]: [280, 283, 284, 285, 277, 285, 282, 283, 270, 284, 284, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892055243Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[0]R generation[0..12]: [164, 167, 168, 169, 164, 169, 166, 167, 159, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892072891Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892087309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[1]R flush_numbers[0..12]: [280, 283, 284, 285, 277, 285, 282, 283, 270, 284, 284, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892100827Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[1]R generation[0..12]: [164, 167, 168, 169, 164, 169, 166, 167, 159, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892114375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892127953Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[2]R flush_numbers[0..12]: [280, 283, 284, 285, 277, 285, 282, 283, 270, 284, 284, 285]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892141361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[2]R generation[0..12]: [164, 167, 168, 169, 164, 169, 166, 167, 159, 168, 168, 169]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.89215459Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892168288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892180966Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892194014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Next flush: 286","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892206612Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892219221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"":"downstairs","":Wait for a query_work_queue command to finish before sending IO "downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892238418Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892250237Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"0","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892264635Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"1","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892277363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"client":"2","":"downstairs","session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892291291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892303409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"76044c40-4a64-4a9e-9ab9-9aa4bb704cae is now active with session: 070b798c-eb28-435d-bf91-e7bc4d43ce5b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892315917Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892327666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862,"session_id":"070b798c-eb28-435d-bf91-e7bc4d43ce5b"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:31.892339594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 327 len 4096 data: 7 02/30 Read block 284 len 40960 03/30 Read block 117 len 32768 04/30 Write block 139 len 36864 data: 9 10 10 14 15 16 18 16 16 05/30 Write block 33 len 4096 data: 11 06/30 Write block 4 len 36864 data: 13 11 11 9 11 13 14 15 13 07/30 Read block 67 len 24576 08/30 Read block 548 len 40960 09/30 Write block 370 len 16384 data: 14 16 15 13 10/30 Write block 516 len 8192 data: 11 11 11/30 Write block 194 len 36864 data: 7 9 8 15 14 14 13 12 11 12/30 Write block 372 len 40960 data: 16 14 16 18 17 19 21 19 17 17 13/30 Write block 400 len 8192 data: 9 10 14/30 Read block 270 len 40960 15/30 Read block 509 len 4096 16/30 Write block 580 len 28672 data: 10 11 15 13 10 9 9 17/30 Write block 119 len 16384 data: 8 10 11 12 18/30 Read block 242 len 40960 19/30 Write block 477 len 20480 data: 16 16 16 13 14 20/30 Read block 293 len 24576 21/30 Read block 529 len 20480 22/30 Write block 553 len 24576 data: 13 16 10 10 9 9 23/30 Read block 61 len 32768 24/30 Read block 120 len 8192 25/30 Read block 511 len 32768 26/30 Read block 495 len 36864 27/30 Write block 232 len 16384 data: 12 12 15 15 28/30 Write block 554 len 8192 data: 17 11 29/30 Read block 502 len 28672 30/30 Read block 166 len 12288 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Write 4 Done Done Done false 1009 Acked Write 2 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Read 10 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 30 29 29 88 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:31.91153465Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 280 280 280 T T T 1 020-039 167 167 167 283 283 283 T T T 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 169 169 169 285 285 285 T T T 6 120-139 166 166 166 282 282 282 T T T 7 140-159 167 167 167 283 283 283 T T T 8 160-179 159 159 159 270 270 270 F F F 9 180-199 168 168 168 284 284 284 T T T 10 200-219 168 168 168 284 284 284 T T T 11 220-239 169 169 169 285 285 285 T T T 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 169 169 169 285 285 285 T T T 17 340-359 166 166 166 282 282 282 F F F 18 360-379 169 169 169 285 285 285 T T T 19 380-399 164 164 164 278 278 278 T T T 20 400-419 166 166 166 282 282 282 T T T 21 420-439 166 166 166 282 282 282 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 161 161 161 274 274 274 T T T 24 480-499 166 166 166 282 282 282 T T T 25 500-519 158 158 158 268 268 268 T T T 26 520-539 164 164 164 280 280 280 F F F 27 540-559 167 167 167 283 283 283 T T T 28 560-579 167 167 167 283 283 283 F F F 29 580-599 167 167 167 283 283 283 T T T Max gen: 169, Max flush: 285 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 170 170 170 286 286 286 F F F 1 020-039 170 170 170 286 286 286 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 170 170 170 286 286 286 F F F 6 120-139 170 170 170 286 286 286 F F F 7 140-159 170 170 170 286 286 286 F F F 8 160-179 159 159 159 270 270 270 F F F 9 180-199 170 170 170 286 286 286 F F F 10 200-219 170 170 170 286 286 286 F F F 11 220-239 170 170 170 286 286 286 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 F F F 14 280-299 156 156 156 265 265 265 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 170 170 170 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 170 170 170 286 286 286 F F F 19 380-399 170 170 170 286 286 286 F F F 20 400-419 170 170 170 286 286 286 F F F 21 420-439 166 166 166 282 282 282 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 170 170 170 286 286 286 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 170 170 170 286 286 286 F F F 26 520-539 164 164 164 280 280 280 F F F 27 540-559 170 170 170 286 286 286 F F F 28 560-579 167 167 167 283 283 283 F F F 29 580-599 170 170 170 286 286 286 F F F Max gen: 170, Max flush: 286 Begin loop 86 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.255396946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.302446123Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.303083946Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.30312555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.303143138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.303157626Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} {"msg":"Crucible 55589d1f-9167-487d-8fd2-ec34001011f5 has session id: 55e705f9-0c66-4936-aa78-26ea30c90e6c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.303687773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Upstairs opts: Upstairs UUID: 55589d1f-9167-487d-8fd2-ec34001011f5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.303733197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Crucible stats registered with UUID: 55589d1f-9167-487d-8fd2-ec34001011f5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.303760733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335495354Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.3355279Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} {"msg":"55589d1f-9167-487d-8fd2-ec34001011f5 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335595431Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335629536Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335662661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335680649Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335697527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335713964Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335730722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.33574709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335764457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335781415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:39901)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335809421Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:42987)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.335841317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:53216)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.336007384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.336579656Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.336638858Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.336671544Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.33669978Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.336727676Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.336755632Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.337263503Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.337303977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.337334913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.337903045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.33794273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.337970836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.337993103Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338017449Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338040626Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338060863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338082351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[0]R flush_numbers[0..12]: [286, 286, 284, 285, 277, 286, 286, 286, 270, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338109417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[0]R generation[0..12]: [170, 170, 168, 169, 164, 170, 170, 170, 159, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338134863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.33816184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[1]R flush_numbers[0..12]: [286, 286, 284, 285, 277, 286, 286, 286, 270, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338185796Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[1]R generation[0..12]: [170, 170, 168, 169, 164, 170, 170, 170, 159, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338212003Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} Wait for a query_work_queue command to finish before sending IO {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338246488Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[2]R flush_numbers[0..12]: [286, 286, 284, 285, 277, 286, 286, 286, 270, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338274794Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[2]R generation[0..12]: [170, 170, 168, 169, 164, 170, 170, 170, 159, 170, 170, 170]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.33830211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338329687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Max found gen is 171","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338355583Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Generation requested: 172 >= found:171","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.33838159Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338410936Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338437682Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"":"downstairs","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338465238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338478666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"0","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338491835Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"1","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338504953Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"client":"2","":"downstairs","session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338517911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338529979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"55589d1f-9167-487d-8fd2-ec34001011f5 is now active with session: 55e705f9-0c66-4936-aa78-26ea30c90e6c","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338542558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338555286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870,"session_id":"55e705f9-0c66-4936-aa78-26ea30c90e6c"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:36.338567154Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 565 len 24576 data: 12 11 11 11 11 9 02/30 Read block 534 len 40960 03/30 Read block 47 len 36864 04/30 Write block 560 len 16384 data: 11 12 12 10 05/30 Read block 132 len 24576 06/30 Write block 378 len 24576 data: 22 20 18 18 15 14 07/30 Write block 128 len 24576 data: 18 21 20 19 18 17 08/30 Write block 375 len 8192 data: 19 18 09/30 Write block 391 len 24576 data: 15 14 16 14 14 12 10/30 Read block 468 len 40960 11/30 Flush 12/30 Write block 234 len 4096 data: 16 13/30 Write block 5 len 36864 data: 12 12 10 12 14 15 16 14 14 14/30 Write block 28 len 28672 data: 15 12 11 6 8 12 13 15/30 Write block 180 len 40960 data: 14 14 17 16 15 10 9 11 9 7 16/30 Flush 17/30 Flush 18/30 Write block 517 len 24576 data: 12 9 8 10 11 13 19/30 Flush 20/30 Write block 118 len 24576 data: 9 9 11 12 13 9 21/30 Write block 427 len 8192 data: 14 15 22/30 Write block 362 len 40960 data: 13 12 11 11 12 15 14 15 15 17 23/30 Read block 375 len 16384 24/30 Flush 25/30 Write block 273 len 36864 data: 15 18 15 19 18 15 12 7 6 26/30 Read block 218 len 40960 27/30 Write block 234 len 24576 data: 17 16 15 17 16 15 28/30 Write block 153 len 32768 data: 8 11 15 16 15 13 12 14 29/30 Read block 343 len 16384 30/30 Read block 562 len 12288 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:36.358621222Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 288 288 288 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 172 172 172 291 291 291 F F F 6 120-139 172 172 172 291 291 291 F F F 7 140-159 170 170 170 286 286 286 T T T 8 160-179 159 159 159 270 270 270 T T T 9 180-199 172 172 172 288 288 288 F F F 10 200-219 170 170 170 286 286 286 F F F 11 220-239 172 172 172 288 288 288 T T T 12 240-259 161 161 161 274 274 274 F F F 13 260-279 166 166 166 282 282 282 T T T 14 280-299 156 156 156 265 265 265 T T T 15 300-319 166 166 166 282 282 282 F F F 16 320-339 170 170 170 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 172 172 172 291 291 291 F F F 19 380-399 172 172 172 287 287 287 F F F 20 400-419 170 170 170 286 286 286 F F F 21 420-439 172 172 172 291 291 291 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 170 170 170 286 286 286 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 172 172 172 290 290 290 F F F 27 540-559 170 170 170 286 286 286 F F F 28 560-579 172 172 172 287 287 287 F F F 29 580-599 170 170 170 286 286 286 F F F Max gen: 172, Max flush: 291 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 172 172 172 288 288 288 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 172 172 172 291 291 291 F F F 6 120-139 172 172 172 291 291 291 F F F 7 140-159 173 173 173 292 292 292 F F F 8 160-179 173 173 173 292 292 292 F F F 9 180-199 172 172 172 288 288 288 F F F 10 200-219 170 170 170 286 286 286 F F F 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 173 173 173 292 292 292 F F F 15 300-319 166 166 166 282 282 282 F F F 16 320-339 170 170 170 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 172 172 172 291 291 291 F F F 19 380-399 172 172 172 287 287 287 F F F 20 400-419 170 170 170 286 286 286 F F F 21 420-439 172 172 172 291 291 291 F F F 22 440-459 166 166 166 282 282 282 F F F 23 460-479 170 170 170 286 286 286 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 172 172 172 290 290 290 F F F 27 540-559 170 170 170 286 286 286 F F F 28 560-579 172 172 172 287 287 287 F F F 29 580-599 170 170 170 286 286 286 F F F Max gen: 173, Max flush: 292 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.634846197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.680028219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.680642385Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.680684629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.680699867Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.680714975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} {"msg":"Crucible 65bfd450-df26-4e55-adad-82fb0a9d6d0e has session id: 21bd548f-85f4-485d-99d6-a0035b9f371b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.68126718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Upstairs opts: Upstairs UUID: 65bfd450-df26-4e55-adad-82fb0a9d6d0e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.681313163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Crucible stats registered with UUID: 65bfd450-df26-4e55-adad-82fb0a9d6d0e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.68133699Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712317534Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71235065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} {"msg":"65bfd450-df26-4e55-adad-82fb0a9d6d0e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71242Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712443817Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712477722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71249622Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712513427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712538404Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.7125657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712583138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712609034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712633401Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_connection connected from Ok(127.0.0.1:61413)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71279025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47094)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712836053Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_connection connected from Ok(127.0.0.1:50068)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.712868579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.713613337Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.713642793Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.713657361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.713670549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.713683427Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.713700145Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.714285825Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.714366684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.714398559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71491036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.714936656Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.714950814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715029513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715057789Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715073247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715086156Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715099884Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[0]R flush_numbers[0..12]: [288, 288, 284, 285, 277, 291, 291, 292, 292, 288, 286, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715113472Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[0]R generation[0..12]: [172, 172, 168, 169, 164, 172, 172, 173, 173, 172, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71512735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715145787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[1]R flush_numbers[0..12]: [288, 288, 284, 285, 277, 291, 291, 292, 292, 288, 286, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715159726Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[1]R generation[0..12]: [172, 172, 168, 169, 164, 172, 172, 173, 173, 172, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715173533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715187282Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[2]R flush_numbers[0..12]: [288, 288, 284, 285, 277, 291, 291, 292, 292, 288, 286, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71520068Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[2]R generation[0..12]: [172, 172, 168, 169, 164, 172, 172, 173, 173, 172, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715214238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715228086Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715240814Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715253742Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Next flush: 293","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715266491Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715279019Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"":"downstairs","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:11:40.715291837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715306505Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"0","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715319513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"1","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715332202Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"client":"2","":"downstairs","session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.71534759Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715360128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"65bfd450-df26-4e55-adad-82fb0a9d6d0e is now active with session: 21bd548f-85f4-485d-99d6-a0035b9f371b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715372926Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715384735Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878,"session_id":"21bd548f-85f4-485d-99d6-a0035b9f371b"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:40.715396733Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 316 len 4096 data: 18 02/30 Write block 544 len 32768 data: 17 17 14 15 13 12 12 13 03/30 Write block 177 len 36864 data: 15 15 13 15 15 18 17 16 11 04/30 Read block 334 len 24576 05/30 Read block 170 len 40960 06/30 Read block 438 len 28672 07/30 Write block 150 len 36864 data: 14 9 8 9 12 16 17 16 14 08/30 Write block 459 len 8192 data: 9 11 09/30 Read block 408 len 20480 10/30 Read block 387 len 40960 11/30 Write block 539 len 40960 data: 15 15 14 12 13 18 18 15 16 14 12/30 Write block 407 len 24576 data: 16 17 18 17 17 19 13/30 Write block 5 len 36864 data: 13 13 11 13 15 16 17 15 15 14/30 Read block 155 len 32768 15/30 Flush 16/30 Write block 448 len 32768 data: 19 20 17 15 16 17 12 11 17/30 Read block 428 len 32768 18/30 Write block 543 len 28672 data: 14 19 19 16 17 15 13 19/30 Write block 149 len 28672 data: 14 15 10 9 10 13 17 20/30 Write block 11 len 4096 data: 18 21/30 Read block 168 len 32768 22/30 Read block 474 len 16384 23/30 Read block 473 len 16384 24/30 Write block 1 len 12288 data: 6 8 10 25/30 Read block 309 len 8192 26/30 Read block 535 len 12288 27/30 Read block 203 len 20480 28/30 Read block 366 len 8192 29/30 Read block 393 len 28672 30/30 Read block 94 len 20480 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 8 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 1 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 15 14 14 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:40.735929096Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 174 174 174 293 293 293 T T T 1 020-039 172 172 172 288 288 288 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 172 172 172 291 291 291 F F F 6 120-139 172 172 172 291 291 291 F F F 7 140-159 174 174 174 293 293 293 T T T 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 F F F 10 200-219 170 170 170 286 286 286 F F F 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 173 173 173 292 292 292 F F F 15 300-319 174 174 174 293 293 293 F F F 16 320-339 170 170 170 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 172 172 172 291 291 291 F F F 19 380-399 172 172 172 287 287 287 F F F 20 400-419 174 174 174 293 293 293 F F F 21 420-439 172 172 172 291 291 291 F F F 22 440-459 174 174 174 293 293 293 T T T 23 460-479 174 174 174 293 293 293 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 174 174 174 293 293 293 F F F 27 540-559 174 174 174 293 293 293 T T T 28 560-579 172 172 172 287 287 287 F F F 29 580-599 170 170 170 286 286 286 F F F Max gen: 174, Max flush: 293 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 172 172 172 288 288 288 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 172 172 172 291 291 291 F F F 6 120-139 172 172 172 291 291 291 F F F 7 140-159 175 175 175 294 294 294 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 F F F 10 200-219 170 170 170 286 286 286 F F F 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 173 173 173 292 292 292 F F F 15 300-319 174 174 174 293 293 293 F F F 16 320-339 170 170 170 286 286 286 F F F 17 340-359 166 166 166 282 282 282 F F F 18 360-379 172 172 172 291 291 291 F F F 19 380-399 172 172 172 287 287 287 F F F 20 400-419 174 174 174 293 293 293 F F F 21 420-439 172 172 172 291 291 291 F F F 22 440-459 175 175 175 294 294 294 F F F 23 460-479 174 174 174 293 293 293 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 174 174 174 293 293 293 F F F 27 540-559 175 175 175 294 294 294 F F F 28 560-579 172 172 172 287 287 287 F F F 29 580-599 170 170 170 286 286 286 F F F Max gen: 175, Max flush: 294 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.009674678Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.054230566Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.054866459Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.054907164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.054922821Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.054937209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} {"msg":"Crucible b637c7cc-559c-4201-b17d-0609ff69e944 has session id: d788db2f-5789-4e5e-96a2-3c3212de81c5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.055569073Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Upstairs opts: Upstairs UUID: b637c7cc-559c-4201-b17d-0609ff69e944, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.05566168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Crucible stats registered with UUID: b637c7cc-559c-4201-b17d-0609ff69e944","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.055695716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.08755032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087582076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} {"msg":"b637c7cc-559c-4201-b17d-0609ff69e944 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087665974Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087689601Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087730265Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.08776943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087803546Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087836461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087869586Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087891063Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087920799Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.087953445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37112)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.08798813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33784)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088024065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_connection connected from Ok(127.0.0.1:65352)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088118743Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.08872389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088761455Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088788231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088814037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088832085Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.088845683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.089415185Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.089445311Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.089465658Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.089968039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.089994336Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090008784Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090096962Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090116319Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090129268Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090141246Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090154074Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[0]R flush_numbers[0..12]: [294, 288, 284, 285, 277, 291, 291, 294, 293, 293, 286, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090167752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[0]R generation[0..12]: [175, 172, 168, 169, 164, 172, 172, 175, 174, 174, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.09018147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090195288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[1]R flush_numbers[0..12]: [294, 288, 284, 285, 277, 291, 291, 294, 293, 293, 286, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090209697Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[1]R generation[0..12]: [175, 172, 168, 169, 164, 172, 172, 175, 174, 174, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090226494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090240492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[2]R flush_numbers[0..12]: [294, 288, 284, 285, 277, 291, 291, 294, 293, 293, 286, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090254031Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[2]R generation[0..12]: [175, 172, 168, 169, 164, 172, 172, 175, 174, 174, 170, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090267479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090280997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090293425Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2025-01-11T06:11:45.090313262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090326021Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090338869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"":"downstairs","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090351707Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090363576Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"0","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090376374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"1","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090389062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"client":"2","":"downstairs","session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.09040179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090415788Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"b637c7cc-559c-4201-b17d-0609ff69e944 is now active with session: d788db2f-5789-4e5e-96a2-3c3212de81c5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090428517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090440195Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886,"session_id":"d788db2f-5789-4e5e-96a2-3c3212de81c5"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:45.090452963Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 324 len 4096 data: 16 02/30 Read block 107 len 24576 03/30 Write block 387 len 4096 data: 17 04/30 Read block 535 len 16384 05/30 Read block 310 len 8192 06/30 Write block 134 len 40960 data: 17 18 14 10 8 10 11 11 15 16 07/30 Read block 576 len 24576 08/30 Write block 588 len 32768 data: 10 11 12 14 13 12 10 10 09/30 Write block 561 len 12288 data: 13 13 11 10/30 Read block 199 len 12288 11/30 Write block 357 len 12288 data: 14 17 15 12/30 Flush 13/30 Write block 418 len 36864 data: 15 15 15 16 14 17 14 17 13 14/30 Read block 293 len 4096 15/30 Write block 25 len 20480 data: 9 9 11 16 13 16/30 Write block 459 len 16384 data: 10 12 12 13 17/30 Read block 520 len 24576 18/30 Read block 583 len 12288 19/30 Write block 340 len 12288 data: 20 18 15 20/30 Read block 253 len 28672 21/30 Read block 579 len 36864 22/30 Write block 586 len 36864 data: 10 9 11 12 13 15 14 13 11 23/30 Write block 208 len 36864 data: 13 12 12 12 12 15 13 12 15 24/30 Read block 567 len 28672 25/30 Read block 261 len 40960 26/30 Write block 388 len 32768 data: 18 14 13 16 15 17 15 15 27/30 Read block 100 len 8192 28/30 Read block 361 len 40960 29/30 Write block 126 len 20480 data: 12 14 19 22 21 30/30 Read block 309 len 12288 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 9 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 16 18 16 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:45.110486444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 172 172 172 288 288 288 T T T 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 172 172 172 291 291 291 F F F 6 120-139 176 176 176 295 295 295 T T T 7 140-159 176 176 176 295 295 295 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 F F F 10 200-219 170 170 170 286 286 286 T T T 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 173 173 173 292 292 292 F F F 15 300-319 174 174 174 293 293 293 F F F 16 320-339 176 176 176 295 295 295 F F F 17 340-359 176 176 176 295 295 295 T T T 18 360-379 172 172 172 291 291 291 F F F 19 380-399 176 176 176 295 295 295 T T T 20 400-419 174 174 174 293 293 293 T T T 21 420-439 172 172 172 291 291 291 T T T 22 440-459 175 175 175 294 294 294 T T T 23 460-479 174 174 174 293 293 293 T T T 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 174 174 174 293 293 293 F F F 27 540-559 175 175 175 294 294 294 F F F 28 560-579 176 176 176 295 295 295 F F F 29 580-599 176 176 176 295 295 295 T T T Max gen: 176, Max flush: 295 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 177 177 177 296 296 296 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 F F F 5 100-119 172 172 172 291 291 291 F F F 6 120-139 177 177 177 296 296 296 F F F 7 140-159 176 176 176 295 295 295 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 F F F 10 200-219 177 177 177 296 296 296 F F F 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 173 173 173 292 292 292 F F F 15 300-319 174 174 174 293 293 293 F F F 16 320-339 176 176 176 295 295 295 F F F 17 340-359 177 177 177 296 296 296 F F F 18 360-379 172 172 172 291 291 291 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 177 177 177 296 296 296 F F F 21 420-439 177 177 177 296 296 296 F F F 22 440-459 177 177 177 296 296 296 F F F 23 460-479 177 177 177 296 296 296 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 174 174 174 293 293 293 F F F 27 540-559 175 175 175 294 294 294 F F F 28 560-579 176 176 176 295 295 295 F F F 29 580-599 177 177 177 296 296 296 F F F Max gen: 177, Max flush: 296 Begin loop 89 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.417622821Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.462724394Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.463371776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.46341184Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.463428198Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.463443306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} {"msg":"Crucible 8bbb2515-729b-4186-ae48-2f156b478f8f has session id: 5ee88fa4-27df-4072-8375-993897420587","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.464007569Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Upstairs opts: Upstairs UUID: 8bbb2515-729b-4186-ae48-2f156b478f8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.464053842Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Crucible stats registered with UUID: 8bbb2515-729b-4186-ae48-2f156b478f8f","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.46407144Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496744543Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496775768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} {"msg":"8bbb2515-729b-4186-ae48-2f156b478f8f active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.49683835Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496865436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496890533Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496905461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496932347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.496956594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.49698242Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497006627Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497029304Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497044342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_connection connected from Ok(127.0.0.1:42332)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497263442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_connection connected from Ok(127.0.0.1:49015)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497301287Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_connection connected from Ok(127.0.0.1:63996)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497327603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.49793454Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.497972225Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.498001041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.498027987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.498056943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.498084959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.498692086Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.498723862Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.49873952Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499266958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499304813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499331609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499352176Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499375703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499399389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499419097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499439844Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[0]R flush_numbers[0..12]: [294, 296, 284, 285, 277, 291, 296, 295, 293, 293, 296, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499462621Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[0]R generation[0..12]: [175, 177, 168, 169, 164, 172, 177, 176, 174, 174, 177, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499487408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499513644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[1]R flush_numbers[0..12]: [294, 296, 284, 285, 277, 291, 296, 295, 293, 293, 296, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499537181Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[1]R generation[0..12]: [175, 177, 168, 169, 164, 172, 177, 176, 174, 174, 177, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499567247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499596683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[2]R flush_numbers[0..12]: [294, 296, 284, 285, 277, 291, 296, 295, 293, 293, 296, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499632678Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[2]R generation[0..12]: [175, 177, 168, 169, 164, 172, 177, 176, 174, 174, 177, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499661314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.49968849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499715276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499741703Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499768589Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499794166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"":"downstairs","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:11:49.499819442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499844439Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"0","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499862016Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"1","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499874864Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"client":"2","":"downstairs","session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499888083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499900111Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"8bbb2515-729b-4186-ae48-2f156b478f8f is now active with session: 5ee88fa4-27df-4072-8375-993897420587","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499915519Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499927817Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894,"session_id":"5ee88fa4-27df-4072-8375-993897420587"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:49.499940036Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 554 len 28672 data: 18 12 11 10 10 8 12 02/30 Read block 117 len 24576 03/30 Read block 528 len 8192 04/30 Write block 367 len 36864 data: 16 15 16 16 18 17 15 17 20 05/30 Write block 551 len 4096 data: 14 06/30 Flush 07/30 Write block 402 len 12288 data: 8 7 6 08/30 Read block 230 len 28672 09/30 Write block 464 len 28672 data: 11 10 9 9 8 10 9 10/30 Read block 72 len 20480 11/30 Read block 366 len 16384 12/30 Write block 200 len 16384 data: 14 13 12 10 13/30 Write block 356 len 36864 data: 15 15 18 16 15 15 14 13 12 14/30 Write block 582 len 12288 data: 16 14 11 15/30 Write block 538 len 28672 data: 10 16 16 15 13 15 20 16/30 Write block 101 len 16384 data: 11 12 9 12 17/30 Write block 426 len 40960 data: 14 15 16 14 18 13 12 11 10 12 18/30 Write block 291 len 24576 data: 18 17 18 14 14 15 19/30 Read block 69 len 32768 20/30 Write block 330 len 40960 data: 8 11 13 15 12 13 16 15 16 18 21/30 Read block 293 len 4096 22/30 Write block 212 len 4096 data: 13 23/30 Read block 81 len 36864 24/30 Read block 484 len 12288 25/30 Read block 579 len 36864 26/30 Write block 587 len 36864 data: 10 12 13 14 16 15 14 12 11 27/30 Read block 377 len 8192 28/30 Read block 266 len 4096 29/30 Write block 313 len 24576 data: 10 16 15 19 20 18 30/30 Write block 98 len 20480 data: 14 13 12 12 13 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 3 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 7 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 4 Done Done Done false 1011 Acked Write 4 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:49.519161498Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 177 177 177 296 296 296 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 164 164 164 277 277 277 T T T 5 100-119 172 172 172 291 291 291 T T T 6 120-139 177 177 177 296 296 296 F F F 7 140-159 176 176 176 295 295 295 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 F F F 10 200-219 177 177 177 296 296 296 T T T 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 173 173 173 292 292 292 T T T 15 300-319 174 174 174 293 293 293 T T T 16 320-339 176 176 176 295 295 295 T T T 17 340-359 177 177 177 296 296 296 T T T 18 360-379 178 178 178 297 297 297 T T T 19 380-399 177 177 177 296 296 296 F F F 20 400-419 177 177 177 296 296 296 T T T 21 420-439 177 177 177 296 296 296 T T T 22 440-459 177 177 177 296 296 296 F F F 23 460-479 177 177 177 296 296 296 T T T 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 174 174 174 293 293 293 T T T 27 540-559 178 178 178 297 297 297 T T T 28 560-579 178 178 178 297 297 297 F F F 29 580-599 177 177 177 296 296 296 T T T Max gen: 178, Max flush: 297 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 177 177 177 296 296 296 F F F 2 040-059 168 168 168 284 284 284 F F F 3 060-079 169 169 169 285 285 285 F F F 4 080-099 179 179 179 298 298 298 F F F 5 100-119 179 179 179 298 298 298 F F F 6 120-139 177 177 177 296 296 296 F F F 7 140-159 176 176 176 295 295 295 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 F F F 10 200-219 179 179 179 298 298 298 F F F 11 220-239 173 173 173 292 292 292 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 179 179 179 298 298 298 F F F 15 300-319 179 179 179 298 298 298 F F F 16 320-339 179 179 179 298 298 298 F F F 17 340-359 179 179 179 298 298 298 F F F 18 360-379 179 179 179 298 298 298 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 179 179 179 298 298 298 F F F 21 420-439 179 179 179 298 298 298 F F F 22 440-459 177 177 177 296 296 296 F F F 23 460-479 179 179 179 298 298 298 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 179 179 179 298 298 298 F F F Max gen: 179, Max flush: 298 Begin loop 90 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.846819727Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.892212221Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.892833936Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.892884479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.892902496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.892919804Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} {"msg":"Crucible b05c92f3-1291-40f3-86e0-e98644f5c244 has session id: fdbe97c0-93b7-4540-a908-60adee7f0f01","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.893478757Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Upstairs opts: Upstairs UUID: b05c92f3-1291-40f3-86e0-e98644f5c244, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.893521582Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Crucible stats registered with UUID: b05c92f3-1291-40f3-86e0-e98644f5c244","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.893539849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.924914029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.924947225Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} {"msg":"b05c92f3-1291-40f3-86e0-e98644f5c244 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925009016Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925041012Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925071048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925093475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925110523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925132029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925160785Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925189602Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925218058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925248493Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_connection connected from Ok(127.0.0.1:51535)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925280389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_connection connected from Ok(127.0.0.1:59302)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.925311415Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_connection connected from Ok(127.0.0.1:43419)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.9253436Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926039985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926068991Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926084179Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926097948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926112126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926125214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926609847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926717573Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.926806591Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927189558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927227683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927256729Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927314661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927341737Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927355836Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927368124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927381392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[0]R flush_numbers[0..12]: [294, 296, 284, 285, 298, 298, 296, 295, 293, 293, 298, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.92739526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[0]R generation[0..12]: [175, 177, 168, 169, 179, 179, 177, 176, 174, 174, 179, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927415167Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927428986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[1]R flush_numbers[0..12]: [294, 296, 284, 285, 298, 298, 296, 295, 293, 293, 298, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927442473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[1]R generation[0..12]: [175, 177, 168, 169, 179, 179, 177, 176, 174, 174, 179, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927458671Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927472629Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[2]R flush_numbers[0..12]: [294, 296, 284, 285, 298, 298, 296, 295, 293, 293, 298, 292]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927486048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[2]R generation[0..12]: [175, 177, 168, 169, 179, 179, 177, 176, 174, 174, 179, 173]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927499496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927513054Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Max found gen is 180","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927534151Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927547059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927559608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927572236Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"":"downstairs","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927585014Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927596683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"0","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927609321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"1","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927622109Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"client":"2","":"downstairs","session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927634977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927648545Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"b05c92f3-1291-40f3-86e0-e98644f5c244 is now active with session: fdbe97c0-93b7-4540-a908-60adee7f0f01","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927661164Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.927672872Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902,"session_id":"fdbe97c0-93b7-4540-a908-60adee7f0f01"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:53.92768466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 594 len 8192 data: 13 12 02/30 Write block 133 len 8192 data: 18 18 03/30 Write block 319 len 24576 data: 17 16 15 16 14 17 04/30 Write block 141 len 24576 data: 12 16 17 17 19 17 05/30 Flush 06/30 Write block 356 len 20480 data: 16 16 19 17 16 07/30 Read block 234 len 12288 08/30 Write block 407 len 36864 data: 17 18 19 18 18 20 15 15 16 09/30 Write block 115 len 36864 data: 9 8 10 10 10 12 13 14 10 10/30 Write block 281 len 16384 data: 7 7 8 9 11/30 Flush 12/30 Write block 56 len 36864 data: 14 15 14 16 20 18 15 14 13 13/30 Write block 308 len 36864 data: 10 9 11 11 8 11 17 16 20 14/30 Read block 257 len 16384 15/30 Read block 306 len 12288 16/30 Write block 314 len 36864 data: 18 17 21 21 19 18 17 16 17 17/30 Write block 301 len 28672 data: 14 13 11 11 8 10 9 18/30 Read block 319 len 28672 19/30 Read block 235 len 20480 20/30 Flush 21/30 Write block 186 len 36864 data: 10 12 10 8 9 7 7 5 8 22/30 Write block 221 len 40960 data: 12 15 14 13 14 13 8 10 11 12 23/30 Read block 192 len 36864 24/30 Read block 236 len 8192 25/30 Write block 141 len 20480 data: 13 17 18 18 20 26/30 Write block 309 len 20480 data: 10 12 12 9 12 27/30 Read block 329 len 20480 28/30 Read block 503 len 28672 29/30 Write block 83 len 8192 data: 12 13 30/30 Read block 480 len 32768 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 9 10 10 29 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:53.946936768Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 177 177 177 296 296 296 F F F 2 040-059 180 180 180 301 301 301 F F F 3 060-079 180 180 180 301 301 301 F F F 4 080-099 179 179 179 298 298 298 T T T 5 100-119 180 180 180 300 300 300 F F F 6 120-139 180 180 180 300 300 300 F F F 7 140-159 180 180 180 299 299 299 T T T 8 160-179 174 174 174 293 293 293 F F F 9 180-199 174 174 174 293 293 293 T T T 10 200-219 179 179 179 298 298 298 F F F 11 220-239 173 173 173 292 292 292 T T T 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 180 180 180 300 300 300 F F F 15 300-319 180 180 180 301 301 301 T T T 16 320-339 180 180 180 301 301 301 F F F 17 340-359 180 180 180 300 300 300 F F F 18 360-379 180 180 180 300 300 300 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 180 180 180 300 300 300 F F F 21 420-439 179 179 179 298 298 298 F F F 22 440-459 177 177 177 296 296 296 F F F 23 460-479 179 179 179 298 298 298 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 180 180 180 299 299 299 F F F Max gen: 180, Max flush: 301 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 F F F 1 020-039 177 177 177 296 296 296 F F F 2 040-059 180 180 180 301 301 301 F F F 3 060-079 180 180 180 301 301 301 F F F 4 080-099 181 181 181 302 302 302 F F F 5 100-119 180 180 180 300 300 300 F F F 6 120-139 180 180 180 300 300 300 F F F 7 140-159 181 181 181 302 302 302 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 181 181 181 302 302 302 F F F 10 200-219 179 179 179 298 298 298 F F F 11 220-239 181 181 181 302 302 302 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 173 173 173 292 292 292 F F F 14 280-299 180 180 180 300 300 300 F F F 15 300-319 181 181 181 302 302 302 F F F 16 320-339 180 180 180 301 301 301 F F F 17 340-359 180 180 180 300 300 300 F F F 18 360-379 180 180 180 300 300 300 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 180 180 180 300 300 300 F F F 21 420-439 179 179 179 298 298 298 F F F 22 440-459 177 177 177 296 296 296 F F F 23 460-479 179 179 179 298 298 298 F F F 24 480-499 170 170 170 286 286 286 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 180 180 180 299 299 299 F F F Max gen: 181, Max flush: 302 Begin loop 91 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.23027247Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.275355295Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.276086995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.27612934Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.276143918Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.276158276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} {"msg":"Crucible 0505d920-73a1-4f30-90fd-6cd0fb23604a has session id: 8f684647-9a7e-46d9-baa1-b2978cb9c262","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.276774361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Upstairs opts: Upstairs UUID: 0505d920-73a1-4f30-90fd-6cd0fb23604a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.276818135Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Crucible stats registered with UUID: 0505d920-73a1-4f30-90fd-6cd0fb23604a","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.276845272Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308140763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308177098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} {"msg":"0505d920-73a1-4f30-90fd-6cd0fb23604a active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308260076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308283813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308312469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308329257Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308345615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308371851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308411986Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308444281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308479256Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308512782Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_connection connected from Ok(127.0.0.1:42465)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308544517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_connection connected from Ok(127.0.0.1:55378)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308576473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_connection connected from Ok(127.0.0.1:51548)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.308607389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.309483949Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.309515975Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.309542361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.309569877Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.309599813Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.30962606Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310152558Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310186363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310309556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310770363Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.31079718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310811578Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310887457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310908664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310922182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310934141Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310947409Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[0]R flush_numbers[0..12]: [294, 296, 301, 301, 302, 300, 300, 302, 293, 302, 298, 302]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310961297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[0]R generation[0..12]: [175, 177, 180, 180, 181, 180, 180, 181, 174, 181, 179, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310974775Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.310988443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[1]R flush_numbers[0..12]: [294, 296, 301, 301, 302, 300, 300, 302, 293, 302, 298, 302]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311001752Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[1]R generation[0..12]: [175, 177, 180, 180, 181, 180, 180, 181, 174, 181, 179, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.31101499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311032267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[2]R flush_numbers[0..12]: [294, 296, 301, 301, 302, 300, 300, 302, 293, 302, 298, 302]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311046046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[2]R generation[0..12]: [175, 177, 180, 180, 181, 180, 180, 181, 174, 181, 179, 181]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311059444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311072912Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.31108527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311098058Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Next flush: 303","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311110457Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311122865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"":"downstairs","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311135513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:11:58.311147281Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"0","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311165669Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"1","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311178367Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"client":"2","":"downstairs","session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311191116Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311202894Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"0505d920-73a1-4f30-90fd-6cd0fb23604a is now active with session: 8f684647-9a7e-46d9-baa1-b2978cb9c262","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311215282Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.31122919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910,"session_id":"8f684647-9a7e-46d9-baa1-b2978cb9c262"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:11:58.311241889Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 495 len 8192 data: 17 16 03/30 Read block 308 len 16384 04/30 Write block 406 len 32768 data: 10 18 19 20 19 19 21 16 05/30 Write block 418 len 16384 data: 16 16 16 17 06/30 Read block 314 len 36864 07/30 Read block 57 len 8192 08/30 Read block 323 len 20480 09/30 Write block 475 len 40960 data: 15 15 17 17 17 14 15 13 15 13 10/30 Write block 272 len 4096 data: 16 11/30 Write block 212 len 24576 data: 14 16 14 13 16 16 12/30 Flush 13/30 Write block 422 len 12288 data: 15 18 15 14/30 Write block 324 len 36864 data: 18 12 10 8 6 8 9 12 14 15/30 Read block 25 len 24576 16/30 Read block 360 len 36864 17/30 Read block 11 len 8192 18/30 Read block 346 len 32768 19/30 Write block 447 len 16384 data: 19 20 21 18 20/30 Write block 423 len 32768 data: 19 16 18 15 16 17 15 19 21/30 Write block 220 len 16384 data: 14 13 16 15 22/30 Write block 472 len 20480 data: 11 13 15 16 16 23/30 Write block 286 len 20480 data: 12 13 15 17 18 24/30 Read block 299 len 12288 25/30 Read block 247 len 40960 26/30 Write block 337 len 8192 data: 16 17 27/30 Write block 16 len 40960 data: 18 16 16 18 16 14 13 12 11 10 28/30 Write block 416 len 16384 data: 15 16 17 17 29/30 Write block 262 len 24576 data: 12 12 16 16 15 13 30/30 Read block 591 len 12288 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:18 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1012 Acked Write 3 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 3 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 17 18 18 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1011 1011 1011 Downstairs last five completed: 1011 1010 1009 1008 1007 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:11:58.330206286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 175 175 175 294 294 294 T T T 1 020-039 177 177 177 296 296 296 T T T 2 040-059 180 180 180 301 301 301 F F F 3 060-079 180 180 180 301 301 301 F F F 4 080-099 181 181 181 302 302 302 F F F 5 100-119 180 180 180 300 300 300 F F F 6 120-139 180 180 180 300 300 300 F F F 7 140-159 181 181 181 302 302 302 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 181 181 181 302 302 302 F F F 10 200-219 182 182 182 304 304 304 F F F 11 220-239 181 181 181 302 302 302 T T T 12 240-259 161 161 161 274 274 274 F F F 13 260-279 182 182 182 304 304 304 T T T 14 280-299 180 180 180 300 300 300 T T T 15 300-319 181 181 181 302 302 302 F F F 16 320-339 180 180 180 301 301 301 T T T 17 340-359 180 180 180 300 300 300 F F F 18 360-379 180 180 180 300 300 300 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 182 182 182 304 304 304 T T T 21 420-439 182 182 182 304 304 304 T T T 22 440-459 177 177 177 296 296 296 T T T 23 460-479 182 182 182 304 304 304 T T T 24 480-499 182 182 182 304 304 304 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 180 180 180 299 299 299 F F F Max gen: 182, Max flush: 304 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 183 183 183 305 305 305 F F F 2 040-059 180 180 180 301 301 301 F F F 3 060-079 180 180 180 301 301 301 F F F 4 080-099 181 181 181 302 302 302 F F F 5 100-119 180 180 180 300 300 300 F F F 6 120-139 180 180 180 300 300 300 F F F 7 140-159 181 181 181 302 302 302 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 181 181 181 302 302 302 F F F 10 200-219 182 182 182 304 304 304 F F F 11 220-239 183 183 183 305 305 305 F F F 12 240-259 161 161 161 274 274 274 F F F 13 260-279 183 183 183 305 305 305 F F F 14 280-299 183 183 183 305 305 305 F F F 15 300-319 181 181 181 302 302 302 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 180 180 180 300 300 300 F F F 18 360-379 180 180 180 300 300 300 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 183 183 183 305 305 305 F F F 22 440-459 183 183 183 305 305 305 F F F 23 460-479 183 183 183 305 305 305 F F F 24 480-499 182 182 182 304 304 304 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 178 178 178 297 297 297 F F F 29 580-599 180 180 180 299 299 299 F F F Max gen: 183, Max flush: 305 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.649172025Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:02.695786131Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.696250328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.729376608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.730137514Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.730178719Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.730195487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.730213204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} {"msg":"Crucible 81b39dd4-fd7b-41d6-91c6-bae8b4dc777b has session id: 0abc9d48-82e3-431a-a96d-4da077e0fecf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.73082603Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Upstairs opts: Upstairs UUID: 81b39dd4-fd7b-41d6-91c6-bae8b4dc777b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.730863435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Crucible stats registered with UUID: 81b39dd4-fd7b-41d6-91c6-bae8b4dc777b","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.730879233Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.762913903Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.762955017Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} {"msg":"81b39dd4-fd7b-41d6-91c6-bae8b4dc777b active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763042595Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763076011Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763108326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763126994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763149231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763175687Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763204163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763232869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763262925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763293521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:33963)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763324317Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:53050)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763355253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_connection connected from Ok(127.0.0.1:38641)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.763386598Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764017852Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764045888Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764067815Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764091792Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764266928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764297124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764770779Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764822822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.764856047Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765397763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765425849Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765440297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765506718Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765526026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765539254Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765551172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.76556412Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[0]R flush_numbers[0..12]: [305, 305, 301, 301, 302, 300, 300, 302, 293, 302, 304, 305]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765577829Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[0]R generation[0..12]: [183, 183, 180, 180, 181, 180, 180, 181, 174, 181, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765591357Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765605155Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[1]R flush_numbers[0..12]: [305, 305, 301, 301, 302, 300, 300, 302, 293, 302, 304, 305]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765618693Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[1]R generation[0..12]: [183, 183, 180, 180, 181, 180, 180, 181, 174, 181, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765635201Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765649209Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[2]R flush_numbers[0..12]: [305, 305, 301, 301, 302, 300, 300, 302, 293, 302, 304, 305]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765663137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[2]R generation[0..12]: [183, 183, 180, 180, 181, 180, 180, 181, 174, 181, 182, 183]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765676715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765690273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765702772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.76571555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:12:02.765728148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765746635Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"":"downstairs","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765759684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765771362Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"0","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.76578442Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"1","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765797259Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"client":"2","":"downstairs","session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765809997Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765821925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"81b39dd4-fd7b-41d6-91c6-bae8b4dc777b is now active with session: 0abc9d48-82e3-431a-a96d-4da077e0fecf","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765836263Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.765848132Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918,"session_id":"0abc9d48-82e3-431a-a96d-4da077e0fecf"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:02.76586007Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 40960 data: 12 10 11 12 17 15 12 10 11 11 02/30 Read block 432 len 24576 03/30 Read block 227 len 20480 04/30 Write block 216 len 40960 data: 17 17 15 15 15 14 17 16 14 15 05/30 Read block 532 len 36864 06/30 Read block 111 len 4096 07/30 Write block 372 len 12288 data: 18 16 18 08/30 Write block 287 len 4096 data: 14 09/30 Read block 417 len 8192 10/30 Read block 385 len 16384 11/30 Write block 494 len 16384 data: 17 18 17 16 12/30 Read block 219 len 12288 13/30 Write block 77 len 16384 data: 10 11 12 11 14/30 Read block 590 len 32768 15/30 Write block 246 len 16384 data: 18 19 20 15 16/30 Flush 17/30 Write block 54 len 24576 data: 11 10 15 16 15 17 18/30 Write block 69 len 8192 data: 13 15 19/30 Write block 304 len 28672 data: 12 9 11 10 11 11 13 20/30 Write block 76 len 36864 data: 12 11 12 13 12 12 12 13 14 21/30 Read block 38 len 4096 22/30 Write block 423 len 24576 data: 20 17 19 16 17 18 23/30 Read block 392 len 24576 24/30 Write block 128 len 4096 data: 20 25/30 Write block 428 len 36864 data: 19 16 20 14 13 12 11 13 14 26/30 Write block 285 len 28672 data: 11 13 15 16 18 19 19 27/30 Write block 51 len 36864 data: 12 10 10 12 11 16 17 16 18 28/30 Read block 174 len 28672 29/30 Write block 346 len 8192 data: 12 13 30/30 Write block 204 len 16384 data: 7 10 11 12 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 6 Done Done Done false 1017 Acked Write 2 Done Done Done false 1018 Acked Write 7 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 2 Sent Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 1 5 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:02.784993673Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 183 183 183 305 305 305 F F F 2 040-059 180 180 180 301 301 301 T T T 3 060-079 184 184 184 306 306 306 T T T 4 080-099 184 184 184 306 306 306 T T T 5 100-119 180 180 180 300 300 300 F F F 6 120-139 180 180 180 300 300 300 T T T 7 140-159 181 181 181 302 302 302 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 181 181 181 302 302 302 F F F 10 200-219 184 184 184 306 306 306 T T T 11 220-239 184 184 184 306 306 306 F F F 12 240-259 184 184 184 306 306 306 F F F 13 260-279 183 183 183 305 305 305 F F F 14 280-299 184 184 184 306 306 306 T T T 15 300-319 181 181 181 302 302 302 T T T 16 320-339 183 183 183 305 305 305 F F F 17 340-359 180 180 180 300 300 300 T T T 18 360-379 184 184 184 306 306 306 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 183 183 183 305 305 305 T T T 22 440-459 183 183 183 305 305 305 F F F 23 460-479 183 183 183 305 305 305 F F F 24 480-499 184 184 184 306 306 306 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 184 184 184 306 306 306 F F F 29 580-599 184 184 184 306 306 306 F F F Max gen: 184, Max flush: 306 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 183 183 183 305 305 305 F F F 2 040-059 185 185 185 307 307 307 F F F 3 060-079 185 185 185 307 307 307 F F F 4 080-099 185 185 185 307 307 307 F F F 5 100-119 180 180 180 300 300 300 F F F 6 120-139 185 185 185 307 307 307 F F F 7 140-159 181 181 181 302 302 302 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 181 181 181 302 302 302 F F F 10 200-219 185 185 185 307 307 307 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 184 184 184 306 306 306 F F F 13 260-279 183 183 183 305 305 305 F F F 14 280-299 185 185 185 307 307 307 F F F 15 300-319 185 185 185 307 307 307 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 185 185 185 307 307 307 F F F 18 360-379 184 184 184 306 306 306 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 185 185 185 307 307 307 F F F 22 440-459 183 183 183 305 305 305 F F F 23 460-479 183 183 183 305 305 305 F F F 24 480-499 184 184 184 306 306 306 F F F 25 500-519 172 172 172 290 290 290 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 184 184 184 306 306 306 F F F 29 580-599 184 184 184 306 306 306 F F F Max gen: 185, Max flush: 307 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.120696515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.167215694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.168074477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.16812524Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.168146147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.168164204Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} {"msg":"Crucible 489a1d1a-dfc3-4d51-90fd-28c17ade6e7e has session id: 90ba6110-5244-409d-9942-2e9da92b2065","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.168742945Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Upstairs opts: Upstairs UUID: 489a1d1a-dfc3-4d51-90fd-28c17ade6e7e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.168822705Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Crucible stats registered with UUID: 489a1d1a-dfc3-4d51-90fd-28c17ade6e7e","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.16885191Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200257236Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200299331Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} {"msg":"489a1d1a-dfc3-4d51-90fd-28c17ade6e7e active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200367032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200389218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200420104Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200437232Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20045374Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200469987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200491594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200507712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20052461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200552306Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_connection connected from Ok(127.0.0.1:33132)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20067039Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_connection connected from Ok(127.0.0.1:50523)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200705285Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_connection connected from Ok(127.0.0.1:64476)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.200730352Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.201559138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.201598923Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.201629019Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.201657625Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.201684341Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.201702559Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202156957Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202184993Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20227943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202757994Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202796839Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202824065Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202840013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202854691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202868509Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202881218Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202895046Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[0]R flush_numbers[0..12]: [305, 305, 307, 307, 307, 300, 307, 302, 293, 302, 307, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202908744Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[0]R generation[0..12]: [183, 183, 185, 185, 185, 180, 185, 181, 174, 181, 185, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202922852Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20293767Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[1]R flush_numbers[0..12]: [305, 305, 307, 307, 307, 300, 307, 302, 293, 302, 307, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202951388Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[1]R generation[0..12]: [183, 183, 185, 185, 185, 180, 185, 181, 174, 181, 185, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202965956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.202990343Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} Wait for a query_work_queue command to finish before sending IO {"msg":"[2]R flush_numbers[0..12]: [305, 305, 307, 307, 307, 300, 307, 302, 293, 302, 307, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203026188Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[2]R generation[0..12]: [183, 183, 185, 185, 185, 180, 185, 181, 174, 181, 185, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203054644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20308294Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203107967Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203133853Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Next flush: 308","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203159599Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203185496Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"":"downstairs","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203212003Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2025-01-11T06:12:07.203236079Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"0","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203265905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"1","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203290142Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"client":"2","":"downstairs","session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.20330517Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203317548Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"489a1d1a-dfc3-4d51-90fd-28c17ade6e7e is now active with session: 90ba6110-5244-409d-9942-2e9da92b2065","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203330806Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203346384Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926,"session_id":"90ba6110-5244-409d-9942-2e9da92b2065"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:07.203359172Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 142 len 36864 data: 18 19 19 21 18 17 16 15 16 02/30 Write block 368 len 40960 data: 16 17 17 19 19 17 19 21 19 20 03/30 Read block 466 len 32768 04/30 Read block 399 len 32768 05/30 Write block 136 len 40960 data: 15 11 9 11 12 14 19 20 20 22 06/30 Read block 321 len 32768 07/30 Read block 255 len 12288 08/30 Flush 09/30 Write block 152 len 28672 data: 10 11 14 18 18 17 15 10/30 Write block 182 len 8192 data: 19 18 11/30 Write block 507 len 12288 data: 8 10 9 12/30 Read block 237 len 32768 13/30 Flush 14/30 Write block 565 len 16384 data: 13 12 12 12 15/30 Write block 475 len 12288 data: 17 17 18 16/30 Read block 341 len 4096 17/30 Write block 203 len 40960 data: 11 8 11 12 13 14 13 13 13 15 18/30 Write block 120 len 28672 data: 13 14 15 11 9 9 13 19/30 Write block 469 len 4096 data: 11 20/30 Read block 478 len 36864 21/30 Read block 19 len 12288 22/30 Read block 197 len 20480 23/30 Read block 370 len 24576 24/30 Read block 206 len 40960 25/30 Write block 129 len 32768 data: 23 22 20 19 19 19 19 16 26/30 Write block 263 len 20480 data: 13 17 17 16 14 27/30 Read block 71 len 40960 28/30 Read block 76 len 28672 29/30 Write block 489 len 36864 data: 16 19 18 16 18 18 19 18 17 30/30 Write block 352 len 24576 data: 10 13 12 14 17 17 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 4 Done Done Done false 1014 Acked Write 3 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Write 7 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:07.225466869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 183 183 183 305 305 305 F F F 2 040-059 185 185 185 307 307 307 F F F 3 060-079 185 185 185 307 307 307 F F F 4 080-099 185 185 185 307 307 307 F F F 5 100-119 180 180 180 300 300 300 F F F 6 120-139 186 186 186 308 308 308 T T T 7 140-159 186 186 186 309 309 309 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 186 186 186 309 309 309 F F F 10 200-219 185 185 185 307 307 307 T T T 11 220-239 184 184 184 306 306 306 F F F 12 240-259 184 184 184 306 306 306 F F F 13 260-279 183 183 183 305 305 305 T T T 14 280-299 185 185 185 307 307 307 F F F 15 300-319 185 185 185 307 307 307 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 185 185 185 307 307 307 T T T 18 360-379 186 186 186 308 308 308 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 185 185 185 307 307 307 F F F 22 440-459 183 183 183 305 305 305 F F F 23 460-479 183 183 183 305 305 305 T T T 24 480-499 184 184 184 306 306 306 T T T 25 500-519 186 186 186 309 309 309 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 184 184 184 306 306 306 T T T 29 580-599 184 184 184 306 306 306 F F F Max gen: 186, Max flush: 309 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 183 183 183 305 305 305 F F F 2 040-059 185 185 185 307 307 307 F F F 3 060-079 185 185 185 307 307 307 F F F 4 080-099 185 185 185 307 307 307 F F F 5 100-119 180 180 180 300 300 300 F F F 6 120-139 187 187 187 310 310 310 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 186 186 186 309 309 309 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 184 184 184 306 306 306 F F F 13 260-279 187 187 187 310 310 310 F F F 14 280-299 185 185 185 307 307 307 F F F 15 300-319 185 185 185 307 307 307 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 187 187 187 310 310 310 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 177 177 177 296 296 296 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 185 185 185 307 307 307 F F F 22 440-459 183 183 183 305 305 305 F F F 23 460-479 187 187 187 310 310 310 F F F 24 480-499 187 187 187 310 310 310 F F F 25 500-519 186 186 186 309 309 309 F F F 26 520-539 179 179 179 298 298 298 F F F 27 540-559 179 179 179 298 298 298 F F F 28 560-579 187 187 187 310 310 310 F F F 29 580-599 184 184 184 306 306 306 F F F Max gen: 187, Max flush: 310 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.521337727Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.566272253Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.56687959Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.56694878Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.566982026Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.56702097Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} {"msg":"Crucible 88019a74-3a1e-45ae-9af2-c8ad0d3ce0f7 has session id: 2f5cd463-2d43-48ee-81b9-6c90bbf5ff59","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.567476728Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Upstairs opts: Upstairs UUID: 88019a74-3a1e-45ae-9af2-c8ad0d3ce0f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.567519892Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Crucible stats registered with UUID: 88019a74-3a1e-45ae-9af2-c8ad0d3ce0f7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.567545948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599354159Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599388175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} {"msg":"88019a74-3a1e-45ae-9af2-c8ad0d3ce0f7 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599460175Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599482182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599511868Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599529226Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599545873Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.59957334Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599604675Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.59964006Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599671456Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599702452Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_connection connected from Ok(127.0.0.1:38455)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599769773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_connection connected from Ok(127.0.0.1:52626)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599845492Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_connection connected from Ok(127.0.0.1:58088)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.599882297Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.60044526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.600474276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.600499113Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.600514071Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.600527479Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.600540657Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601055857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601154633Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.60117666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601600952Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601640037Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601668003Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601732004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.60176021Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601775138Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601788126Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601801905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[0]R flush_numbers[0..12]: [305, 305, 307, 307, 307, 300, 310, 309, 293, 309, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601815893Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[0]R generation[0..12]: [183, 183, 185, 185, 185, 180, 187, 186, 174, 186, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601829401Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601843129Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[1]R flush_numbers[0..12]: [305, 305, 307, 307, 307, 300, 310, 309, 293, 309, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601856347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[1]R generation[0..12]: [183, 183, 185, 185, 185, 180, 187, 186, 174, 186, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601869686Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601883213Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[2]R flush_numbers[0..12]: [305, 305, 307, 307, 307, 300, 310, 309, 293, 309, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601900611Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[2]R generation[0..12]: [183, 183, 185, 185, 185, 180, 187, 186, 174, 186, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601914529Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601928338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601948985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601962163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601974851Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.601987509Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"":"downstairs","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602003777Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602025234Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"0","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602039672Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"1","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.60205245Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"client":"2","":"downstairs","session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602065359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602077287Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"88019a74-3a1e-45ae-9af2-c8ad0d3ce0f7 is now active with session: 2f5cd463-2d43-48ee-81b9-6c90bbf5ff59","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602089915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602101594Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934,"session_id":"2f5cd463-2d43-48ee-81b9-6c90bbf5ff59"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:11.602113572Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 258 len 32768 data: 15 13 12 11 13 14 18 18 02/30 Write block 261 len 36864 data: 12 14 15 19 19 17 15 13 12 03/30 Flush 04/30 Write block 388 len 36864 data: 19 15 14 17 16 18 16 16 13 05/30 Read block 273 len 8192 06/30 Read block 376 len 12288 07/30 Read block 507 len 24576 08/30 Write block 84 len 28672 data: 15 10 10 9 10 13 16 09/30 Read block 299 len 12288 10/30 Write block 305 len 28672 data: 10 12 11 12 12 14 13 11/30 Flush 12/30 Write block 529 len 32768 data: 14 14 12 12 12 15 14 10 13/30 Write block 542 len 4096 data: 14 14/30 Flush 15/30 Write block 184 len 32768 data: 17 12 11 13 11 9 10 8 16/30 Read block 232 len 36864 17/30 Write block 517 len 12288 data: 13 10 9 18/30 Read block 392 len 4096 19/30 Write block 528 len 12288 data: 16 15 15 20/30 Write block 31 len 8192 data: 7 9 21/30 Write block 264 len 8192 data: 20 20 22/30 Read block 339 len 36864 23/30 Flush 24/30 Write block 188 len 16384 data: 12 10 11 9 25/30 Read block 13 len 8192 26/30 Write block 108 len 32768 data: 12 11 11 13 12 13 11 10 27/30 Read block 232 len 8192 28/30 Write block 424 len 20480 data: 18 20 17 18 20 29/30 Read block 177 len 40960 30/30 Write block 298 len 36864 data: 19 19 17 15 14 12 13 11 13 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:11.62179958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 188 188 188 314 314 314 F F F 2 040-059 185 185 185 307 307 307 F F F 3 060-079 185 185 185 307 307 307 F F F 4 080-099 188 188 188 312 312 312 F F F 5 100-119 180 180 180 300 300 300 T T T 6 120-139 187 187 187 310 310 310 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 188 188 188 314 314 314 T T T 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 188 188 188 311 311 311 F F F 13 260-279 188 188 188 314 314 314 F F F 14 280-299 185 185 185 307 307 307 T T T 15 300-319 188 188 188 312 312 312 T T T 16 320-339 183 183 183 305 305 305 F F F 17 340-359 187 187 187 310 310 310 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 188 188 188 312 312 312 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 185 185 185 307 307 307 T T T 22 440-459 183 183 183 305 305 305 F F F 23 460-479 187 187 187 310 310 310 F F F 24 480-499 187 187 187 310 310 310 F F F 25 500-519 188 188 188 314 314 314 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 188 188 188 313 313 313 F F F 28 560-579 187 187 187 310 310 310 F F F 29 580-599 184 184 184 306 306 306 F F F Max gen: 188, Max flush: 314 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 188 188 188 314 314 314 F F F 2 040-059 185 185 185 307 307 307 F F F 3 060-079 185 185 185 307 307 307 F F F 4 080-099 188 188 188 312 312 312 F F F 5 100-119 189 189 189 315 315 315 F F F 6 120-139 187 187 187 310 310 310 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 174 174 174 293 293 293 F F F 9 180-199 189 189 189 315 315 315 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 188 188 188 311 311 311 F F F 13 260-279 188 188 188 314 314 314 F F F 14 280-299 189 189 189 315 315 315 F F F 15 300-319 189 189 189 315 315 315 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 187 187 187 310 310 310 F F F 18 360-379 186 186 186 308 308 308 F F F 19 380-399 188 188 188 312 312 312 F F F 20 400-419 183 183 183 305 305 305 F F F 21 420-439 189 189 189 315 315 315 F F F 22 440-459 183 183 183 305 305 305 F F F 23 460-479 187 187 187 310 310 310 F F F 24 480-499 187 187 187 310 310 310 F F F 25 500-519 188 188 188 314 314 314 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 188 188 188 313 313 313 F F F 28 560-579 187 187 187 310 310 310 F F F 29 580-599 184 184 184 306 306 306 F F F Max gen: 189, Max flush: 315 Begin loop 95 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.899865361Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.945055342Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.945800461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.945842805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.945857273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.945872171Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} {"msg":"Crucible bca5f088-1632-484f-b1e8-d105a826586d has session id: 08519cff-17e6-4761-82ce-2b27a9fa8967","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.946366483Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Upstairs opts: Upstairs UUID: bca5f088-1632-484f-b1e8-d105a826586d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.946403588Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Crucible stats registered with UUID: bca5f088-1632-484f-b1e8-d105a826586d","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.946419866Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.977796716Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.97783787Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} {"msg":"bca5f088-1632-484f-b1e8-d105a826586d active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.97791267Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.977939636Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.977964763Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.977979181Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.977992619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978005467Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978023715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978046062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978069379Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978092475Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_connection connected from Ok(127.0.0.1:37831)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.97813477Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_connection connected from Ok(127.0.0.1:42255)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978158756Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_connection connected from Ok(127.0.0.1:58869)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978179743Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978957907Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978985613Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.978999831Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.979021408Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.979044805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.979066032Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.979546257Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.979575902Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.979682548Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980125107Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980153474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980168272Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980184609Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980199507Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980212526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980224484Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980237392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[0]R flush_numbers[0..12]: [305, 314, 307, 307, 312, 315, 310, 309, 293, 315, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.98025098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[0]R generation[0..12]: [183, 188, 185, 185, 188, 189, 187, 186, 174, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980264928Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980278526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[1]R flush_numbers[0..12]: [305, 314, 307, 307, 312, 315, 310, 309, 293, 315, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980291815Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[1]R generation[0..12]: [183, 188, 185, 185, 188, 189, 187, 186, 174, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980305943Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980319561Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[2]R flush_numbers[0..12]: [305, 314, 307, 307, 312, 315, 310, 309, 293, 315, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980332919Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[2]R generation[0..12]: [183, 188, 185, 185, 188, 189, 187, 186, 174, 189, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980346137Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980359685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980374843Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Generation requested: 190 >= found:190","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2025-01-11T06:12:15.98039441Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Next flush: 316","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980407219Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980419677Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"":"downstairs","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980432445Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980444264Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"0","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980457002Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"1","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.98046977Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"client":"2","":"downstairs","session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980483469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980495527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"bca5f088-1632-484f-b1e8-d105a826586d is now active with session: 08519cff-17e6-4761-82ce-2b27a9fa8967","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980508035Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980519753Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1942,"session_id":"08519cff-17e6-4761-82ce-2b27a9fa8967"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:15.980535661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 381 len 36864 data: 19 16 15 17 19 20 18 20 16 02/30 Write block 77 len 8192 data: 12 13 03/30 Write block 125 len 36864 data: 10 14 15 21 24 23 21 20 20 04/30 Read block 23 len 20480 05/30 Write block 316 len 12288 data: 22 22 20 06/30 Flush 07/30 Write block 124 len 40960 data: 10 11 15 16 22 25 24 22 21 21 08/30 Write block 240 len 24576 data: 14 15 13 13 16 16 09/30 Write block 407 len 8192 data: 19 20 10/30 Read block 57 len 12288 11/30 Write block 426 len 40960 data: 18 19 21 17 21 15 14 13 12 14 12/30 Read block 480 len 28672 13/30 Read block 345 len 16384 14/30 Write block 161 len 28672 data: 11 11 11 9 9 10 13 15/30 Read block 159 len 20480 16/30 Read block 373 len 16384 17/30 Write block 492 len 28672 data: 17 19 19 20 19 18 14 18/30 Write block 377 len 16384 data: 21 23 21 19 19/30 Read block 305 len 12288 20/30 Write block 455 len 28672 data: 12 11 12 13 11 13 13 21/30 Write block 195 len 20480 data: 10 9 16 15 15 22/30 Write block 52 len 32768 data: 11 11 13 12 17 18 17 19 23/30 Read block 445 len 36864 24/30 Read block 534 len 32768 25/30 Read block 328 len 36864 26/30 Read block 554 len 4096 27/30 Write block 78 len 8192 data: 14 14 28/30 Write block 560 len 24576 data: 13 14 14 12 10 14 29/30 Write block 396 len 36864 data: 14 16 13 11 10 11 9 8 7 30/30 Read block 534 len 40960 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 10 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 3 Done Done Done false 1010 Acked Write 10 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 4 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 24 23 23 70 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:16.00080482Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 188 188 188 314 314 314 F F F 2 040-059 185 185 185 307 307 307 T T T 3 060-079 190 190 190 316 316 316 T T T 4 080-099 188 188 188 312 312 312 F F F 5 100-119 189 189 189 315 315 315 F F F 6 120-139 190 190 190 316 316 316 T T T 7 140-159 186 186 186 309 309 309 F F F 8 160-179 174 174 174 293 293 293 T T T 9 180-199 189 189 189 315 315 315 T T T 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 188 188 188 311 311 311 T T T 13 260-279 188 188 188 314 314 314 F F F 14 280-299 189 189 189 315 315 315 F F F 15 300-319 190 190 190 316 316 316 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 187 187 187 310 310 310 F F F 18 360-379 186 186 186 308 308 308 T T T 19 380-399 190 190 190 316 316 316 T T T 20 400-419 183 183 183 305 305 305 T T T 21 420-439 189 189 189 315 315 315 T T T 22 440-459 183 183 183 305 305 305 T T T 23 460-479 187 187 187 310 310 310 T T T 24 480-499 187 187 187 310 310 310 T T T 25 500-519 188 188 188 314 314 314 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 188 188 188 313 313 313 F F F 28 560-579 187 187 187 310 310 310 T T T 29 580-599 184 184 184 306 306 306 F F F Max gen: 190, Max flush: 316 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 188 188 188 314 314 314 F F F 2 040-059 191 191 191 317 317 317 F F F 3 060-079 191 191 191 317 317 317 F F F 4 080-099 188 188 188 312 312 312 F F F 5 100-119 189 189 189 315 315 315 F F F 6 120-139 191 191 191 317 317 317 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 191 191 191 317 317 317 F F F 9 180-199 191 191 191 317 317 317 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 191 191 191 317 317 317 F F F 13 260-279 188 188 188 314 314 314 F F F 14 280-299 189 189 189 315 315 315 F F F 15 300-319 190 190 190 316 316 316 F F F 16 320-339 183 183 183 305 305 305 F F F 17 340-359 187 187 187 310 310 310 F F F 18 360-379 191 191 191 317 317 317 F F F 19 380-399 191 191 191 317 317 317 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 191 191 191 317 317 317 F F F 22 440-459 191 191 191 317 317 317 F F F 23 460-479 191 191 191 317 317 317 F F F 24 480-499 191 191 191 317 317 317 F F F 25 500-519 188 188 188 314 314 314 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 188 188 188 313 313 313 F F F 28 560-579 191 191 191 317 317 317 F F F 29 580-599 184 184 184 306 306 306 F F F Max gen: 191, Max flush: 317 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.328190148Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:20.372093695Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.372486671Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.404958771Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.405523604Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.405558979Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.405577397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.405592715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} {"msg":"Crucible 04296b67-b2dc-4a15-9b2d-9b88b01b98f3 has session id: dd15c4b6-2f96-457c-853b-902b342d3ab7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.406113624Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Upstairs opts: Upstairs UUID: 04296b67-b2dc-4a15-9b2d-9b88b01b98f3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.406154698Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Crucible stats registered with UUID: 04296b67-b2dc-4a15-9b2d-9b88b01b98f3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.406173805Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437354722Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437386837Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} {"msg":"04296b67-b2dc-4a15-9b2d-9b88b01b98f3 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437446669Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437474356Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437499532Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.43751347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437529328Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437542166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437555414Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437572182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.43758619Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437604448Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60180)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437692995Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_connection connected from Ok(127.0.0.1:39708)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437725131Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_connection connected from Ok(127.0.0.1:36054)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.437898458Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.438532571Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.438575375Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.4386098Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.438640856Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.438668372Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.43868494Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439212428Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439252322Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439281758Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439790579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439818615Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439833383Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439845941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439859469Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439872397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439884356Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439897124Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[0]R flush_numbers[0..12]: [305, 314, 317, 317, 312, 315, 317, 309, 317, 317, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439910802Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[0]R generation[0..12]: [183, 188, 191, 191, 188, 189, 191, 186, 191, 191, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.43992435Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439938239Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[1]R flush_numbers[0..12]: [305, 314, 317, 317, 312, 315, 317, 309, 317, 317, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439951847Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[1]R generation[0..12]: [183, 188, 191, 191, 188, 189, 191, 186, 191, 191, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439965265Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.439978913Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[2]R flush_numbers[0..12]: [305, 314, 317, 317, 312, 315, 317, 309, 317, 317, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440009819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[2]R generation[0..12]: [183, 188, 191, 191, 188, 189, 191, 186, 191, 191, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440027776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440041754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2025-01-11T06:12:20.440054443Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.44007298Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Next flush: 318","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440085838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440098357Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"":"downstairs","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440111045Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440122823Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"0","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440135461Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"1","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.44014819Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"client":"2","":"downstairs","session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440161128Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440173076Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"04296b67-b2dc-4a15-9b2d-9b88b01b98f3 is now active with session: dd15c4b6-2f96-457c-853b-902b342d3ab7","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440185595Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440197293Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950,"session_id":"dd15c4b6-2f96-457c-853b-902b342d3ab7"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:20.440209182Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 422 len 32768 data: 16 21 19 21 19 20 22 18 02/30 Flush 03/30 Write block 587 len 36864 data: 12 13 14 15 17 16 15 14 13 04/30 Write block 293 len 32768 data: 19 15 15 16 18 20 20 18 05/30 Read block 106 len 4096 06/30 Read block 16 len 24576 07/30 Write block 188 len 36864 data: 13 11 12 10 8 6 9 11 10 08/30 Flush 09/30 Write block 51 len 28672 data: 13 12 12 14 13 18 19 10/30 Write block 272 len 16384 data: 17 16 19 16 11/30 Write block 337 len 8192 data: 17 18 12/30 Read block 43 len 20480 13/30 Write block 36 len 28672 data: 14 14 13 14 15 20 21 14/30 Write block 24 len 16384 data: 12 11 10 12 15/30 Write block 170 len 4096 data: 12 16/30 Read block 163 len 8192 17/30 Write block 505 len 20480 data: 11 10 9 11 10 18/30 Write block 317 len 4096 data: 23 19/30 Read block 110 len 32768 20/30 Flush 21/30 Write block 577 len 4096 data: 11 22/30 Flush 23/30 Write block 41 len 24576 data: 21 22 20 21 20 20 24/30 Write block 256 len 40960 data: 14 13 16 14 13 13 15 16 21 21 25/30 Read block 404 len 16384 26/30 Write block 325 len 24576 data: 13 11 9 7 9 10 27/30 Write block 321 len 32768 data: 17 18 15 19 14 12 10 8 28/30 Write block 57 len 24576 data: 20 18 20 21 19 16 29/30 Write block 561 len 36864 data: 15 15 13 11 15 13 13 13 12 30/30 Write block 434 len 20480 data: 13 15 15 10 9 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 6 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 8 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:20.45857392Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 192 192 192 320 320 320 F F F 2 040-059 192 192 192 320 320 320 T T T 3 060-079 191 191 191 317 317 317 T T T 4 080-099 188 188 188 312 312 312 F F F 5 100-119 189 189 189 315 315 315 F F F 6 120-139 191 191 191 317 317 317 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 192 192 192 320 320 320 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 191 191 191 317 317 317 T T T 13 260-279 192 192 192 320 320 320 T T T 14 280-299 192 192 192 319 319 319 F F F 15 300-319 192 192 192 320 320 320 F F F 16 320-339 192 192 192 320 320 320 T T T 17 340-359 187 187 187 310 310 310 F F F 18 360-379 191 191 191 317 317 317 F F F 19 380-399 191 191 191 317 317 317 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 192 192 192 318 318 318 T T T 22 440-459 191 191 191 317 317 317 F F F 23 460-479 191 191 191 317 317 317 F F F 24 480-499 191 191 191 317 317 317 F F F 25 500-519 192 192 192 320 320 320 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 188 188 188 313 313 313 F F F 28 560-579 192 192 192 321 321 321 T T T 29 580-599 192 192 192 319 319 319 F F F Max gen: 192, Max flush: 321 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 F F F 1 020-039 192 192 192 320 320 320 F F F 2 040-059 193 193 193 322 322 322 F F F 3 060-079 193 193 193 322 322 322 F F F 4 080-099 188 188 188 312 312 312 F F F 5 100-119 189 189 189 315 315 315 F F F 6 120-139 191 191 191 317 317 317 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 192 192 192 320 320 320 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 192 192 192 320 320 320 F F F 16 320-339 193 193 193 322 322 322 F F F 17 340-359 187 187 187 310 310 310 F F F 18 360-379 191 191 191 317 317 317 F F F 19 380-399 191 191 191 317 317 317 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 193 193 193 322 322 322 F F F 22 440-459 191 191 191 317 317 317 F F F 23 460-479 191 191 191 317 317 317 F F F 24 480-499 191 191 191 317 317 317 F F F 25 500-519 192 192 192 320 320 320 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 188 188 188 313 313 313 F F F 28 560-579 193 193 193 322 322 322 F F F 29 580-599 192 192 192 319 319 319 F F F Max gen: 193, Max flush: 322 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.78273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.827869238Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.828478865Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.828518059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.828533497Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.828552634Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} {"msg":"Crucible f68f0cc2-c3a6-4f31-9f3a-0da1f7387d28 has session id: 2bc0deed-9aa8-4209-8840-a5d12fd9ea35","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.829059555Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Upstairs opts: Upstairs UUID: f68f0cc2-c3a6-4f31-9f3a-0da1f7387d28, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.829113838Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Crucible stats registered with UUID: f68f0cc2-c3a6-4f31-9f3a-0da1f7387d28","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.829143954Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860463801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860496417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} {"msg":"f68f0cc2-c3a6-4f31-9f3a-0da1f7387d28 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860564857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860587084Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860624579Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860656785Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.86068869Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860724925Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860759941Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860793466Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860828001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.860964773Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_connection connected from Ok(127.0.0.1:62397)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861052231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_connection connected from Ok(127.0.0.1:33292)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861078807Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_connection connected from Ok(127.0.0.1:55464)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861097645Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861583708Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861612664Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861627062Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.86164004Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861889776Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.861913863Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.862390438Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.862418684Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.862573523Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.862991406Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863019772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863037309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863058326Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863081023Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863095521Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.86310754Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863120608Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[0]R flush_numbers[0..12]: [305, 320, 322, 322, 312, 315, 317, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863134236Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[0]R generation[0..12]: [183, 192, 193, 193, 188, 189, 191, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863148084Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863161772Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[1]R flush_numbers[0..12]: [305, 320, 322, 322, 312, 315, 317, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.86317527Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[1]R generation[0..12]: [183, 192, 193, 193, 188, 189, 191, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863188748Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863202487Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[2]R flush_numbers[0..12]: [305, 320, 322, 322, 312, 315, 317, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863215815Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[2]R generation[0..12]: [183, 192, 193, 193, 188, 189, 191, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863229163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863242641Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863258269Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863271577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 323","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863292474Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863315321Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"":"downstairs","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863334679Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863349166Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"0","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863371914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"1","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863388801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"client":"2","":"downstairs","session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863413958Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863432915Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"f68f0cc2-c3a6-4f31-9f3a-0da1f7387d28 is now active with session: 2bc0deed-9aa8-4209-8840-a5d12fd9ea35","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863446003Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.863457682Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958,"session_id":"2bc0deed-9aa8-4209-8840-a5d12fd9ea35"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:24.86346976Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 380 len 28672 data: 20 20 17 16 18 20 21 04/30 Write block 548 len 16384 data: 16 14 13 15 05/30 Read block 106 len 16384 06/30 Flush 07/30 Write block 301 len 20480 data: 16 15 13 14 12 08/30 Read block 445 len 4096 09/30 Write block 551 len 32768 data: 16 15 14 19 13 12 11 11 10/30 Read block 34 len 4096 11/30 Write block 332 len 4096 data: 15 12/30 Read block 183 len 4096 13/30 Flush 14/30 Write block 72 len 12288 data: 16 16 14 15/30 Write block 428 len 16384 data: 23 19 22 16 16/30 Read block 435 len 16384 17/30 Write block 385 len 24576 data: 21 22 19 21 17 15 18/30 Write block 509 len 32768 data: 11 8 9 9 12 13 11 12 19/30 Flush 20/30 Write block 487 len 16384 data: 16 20 17 20 21/30 Write block 471 len 36864 data: 10 12 14 16 18 18 19 18 18 22/30 Read block 430 len 20480 23/30 Read block 389 len 40960 24/30 Flush 25/30 Write block 91 len 20480 data: 14 16 14 12 12 26/30 Write block 18 len 4096 data: 17 27/30 Read block 350 len 24576 28/30 Read block 593 len 8192 29/30 Read block 282 len 28672 30/30 Write block 351 len 16384 data: 10 11 14 13 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:24.881579554Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 183 183 183 305 305 305 T T T 1 020-039 192 192 192 320 320 320 F F F 2 040-059 193 193 193 322 322 322 F F F 3 060-079 194 194 194 327 327 327 F F F 4 080-099 188 188 188 312 312 312 T T T 5 100-119 189 189 189 315 315 315 F F F 6 120-139 191 191 191 317 317 317 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 192 192 192 320 320 320 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 194 194 194 326 326 326 F F F 16 320-339 194 194 194 326 326 326 F F F 17 340-359 187 187 187 310 310 310 T T T 18 360-379 191 191 191 317 317 317 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 194 194 194 327 327 327 F F F 22 440-459 191 191 191 317 317 317 F F F 23 460-479 194 194 194 328 328 328 F F F 24 480-499 194 194 194 328 328 328 F F F 25 500-519 194 194 194 327 327 327 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 194 194 194 326 326 326 F F F 28 560-579 193 193 193 322 322 322 F F F 29 580-599 192 192 192 319 319 319 F F F Max gen: 194, Max flush: 328 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 329 329 329 F F F 1 020-039 192 192 192 320 320 320 F F F 2 040-059 193 193 193 322 322 322 F F F 3 060-079 194 194 194 327 327 327 F F F 4 080-099 195 195 195 329 329 329 F F F 5 100-119 189 189 189 315 315 315 F F F 6 120-139 191 191 191 317 317 317 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 192 192 192 320 320 320 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 194 194 194 326 326 326 F F F 16 320-339 194 194 194 326 326 326 F F F 17 340-359 195 195 195 329 329 329 F F F 18 360-379 191 191 191 317 317 317 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 194 194 194 327 327 327 F F F 22 440-459 191 191 191 317 317 317 F F F 23 460-479 194 194 194 328 328 328 F F F 24 480-499 194 194 194 328 328 328 F F F 25 500-519 194 194 194 327 327 327 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 194 194 194 326 326 326 F F F 28 560-579 193 193 193 322 322 322 F F F 29 580-599 192 192 192 319 319 319 F F F Max gen: 195, Max flush: 329 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.152629366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:29.199523694Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.199908261Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.232576115Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233177083Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233221146Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233238114Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233255822Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} {"msg":"Crucible 2feee796-877e-4ff3-9ec0-263831bf0da1 has session id: 9be894b2-b2c4-4e65-8534-6c655f850bf4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233774661Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Upstairs opts: Upstairs UUID: 2feee796-877e-4ff3-9ec0-263831bf0da1, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233832033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Crucible stats registered with UUID: 2feee796-877e-4ff3-9ec0-263831bf0da1","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.233863389Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.264987553Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265019809Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} {"msg":"2feee796-877e-4ff3-9ec0-263831bf0da1 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265068562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265088709Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265116095Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265133273Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.2651581Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265188286Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265218931Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265250127Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265282133Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265311359Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:63763)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265342034Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:55951)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.26537914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38618)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.265558275Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266086033Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266115059Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266130187Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266143905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266156873Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266170351Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266787987Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266828761Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.266897942Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267367168Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267394364Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267409552Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267422Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267435288Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267452666Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267465714Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267478762Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[0]R flush_numbers[0..12]: [329, 320, 322, 327, 329, 315, 317, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267492211Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[0]R generation[0..12]: [195, 192, 193, 194, 195, 189, 191, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267505529Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267519147Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[1]R flush_numbers[0..12]: [329, 320, 322, 327, 329, 315, 317, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267532325Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[1]R generation[0..12]: [195, 192, 193, 194, 195, 189, 191, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267545473Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267559001Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[2]R flush_numbers[0..12]: [329, 320, 322, 327, 329, 315, 317, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267572309Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"[2]R generation[0..12]: [195, 192, 193, 194, 195, 189, 191, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267585658Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267645659Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267663347Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267678715Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267691513Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267704231Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"":"downstairs","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.26771705Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267728968Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"0","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267741996Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"1","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267754875Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"client":"2","":"downstairs","session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267768013Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267780041Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"2feee796-877e-4ff3-9ec0-263831bf0da1 is now active with session: 9be894b2-b2c4-4e65-8534-6c655f850bf4","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.26779255Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267804298Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966,"session_id":"9be894b2-b2c4-4e65-8534-6c655f850bf4"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:29.267816186Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 422 len 20480 data: 17 22 20 22 20 02/30 Read block 279 len 36864 03/30 Read block 426 len 28672 04/30 Write block 368 len 36864 data: 17 18 18 20 20 18 20 22 20 05/30 Write block 569 len 8192 data: 13 10 06/30 Write block 455 len 4096 data: 13 07/30 Read block 194 len 28672 08/30 Read block 569 len 32768 09/30 Write block 430 len 20480 data: 23 17 15 14 14 10/30 Write block 88 len 12288 data: 11 14 17 11/30 Read block 510 len 28672 12/30 Write block 573 len 32768 data: 9 8 7 12 12 13 11 12 13/30 Read block 243 len 24576 14/30 Read block 106 len 20480 15/30 Write block 500 len 20480 data: 12 11 9 10 10 16/30 Write block 72 len 28672 data: 17 17 15 12 13 13 15 17/30 Write block 502 len 12288 data: 10 11 11 18/30 Flush 19/30 Flush 20/30 Write block 119 len 20480 data: 11 14 15 16 12 21/30 Read block 238 len 32768 22/30 Read block 290 len 36864 23/30 Write block 594 len 20480 data: 15 14 9 6 5 24/30 Read block 111 len 20480 25/30 Write block 475 len 12288 data: 19 19 20 26/30 Read block 274 len 40960 27/30 Read block 294 len 36864 28/30 Write block 13 len 24576 data: 16 18 20 19 17 18 29/30 Write block 563 len 12288 data: 14 12 16 30/30 Write block 133 len 16384 data: 22 20 20 17 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 5 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Write 3 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:29.287309411Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 195 195 195 329 329 329 T T T 1 020-039 192 192 192 320 320 320 F F F 2 040-059 193 193 193 322 322 322 F F F 3 060-079 196 196 196 330 330 330 F F F 4 080-099 196 196 196 330 330 330 F F F 5 100-119 189 189 189 315 315 315 T T T 6 120-139 191 191 191 317 317 317 T T T 7 140-159 186 186 186 309 309 309 F F F 8 160-179 192 192 192 320 320 320 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 194 194 194 326 326 326 F F F 16 320-339 194 194 194 326 326 326 F F F 17 340-359 195 195 195 329 329 329 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 196 196 196 330 330 330 F F F 22 440-459 196 196 196 330 330 330 F F F 23 460-479 194 194 194 328 328 328 T T T 24 480-499 194 194 194 328 328 328 F F F 25 500-519 196 196 196 330 330 330 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 194 194 194 326 326 326 F F F 28 560-579 196 196 196 330 330 330 T T T 29 580-599 196 196 196 330 330 330 T T T Max gen: 196, Max flush: 330 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 331 331 331 F F F 1 020-039 192 192 192 320 320 320 F F F 2 040-059 193 193 193 322 322 322 F F F 3 060-079 196 196 196 330 330 330 F F F 4 080-099 196 196 196 330 330 330 F F F 5 100-119 197 197 197 331 331 331 F F F 6 120-139 197 197 197 331 331 331 F F F 7 140-159 186 186 186 309 309 309 F F F 8 160-179 192 192 192 320 320 320 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 192 192 192 319 319 319 F F F 15 300-319 194 194 194 326 326 326 F F F 16 320-339 194 194 194 326 326 326 F F F 17 340-359 195 195 195 329 329 329 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 196 196 196 330 330 330 F F F 22 440-459 196 196 196 330 330 330 F F F 23 460-479 197 197 197 331 331 331 F F F 24 480-499 194 194 194 328 328 328 F F F 25 500-519 196 196 196 330 330 330 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 194 194 194 326 326 326 F F F 28 560-579 197 197 197 331 331 331 F F F 29 580-599 197 197 197 331 331 331 F F F Max gen: 197, Max flush: 331 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.603316226Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.648907712Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.649571781Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"afbd0ec341edbe8e076286ae0030d2abadb28437\",\n git_commit_timestamp: \"2025-01-11T05:49:56.000000000Z\",\n git_branch: \"renovate/opentelemetry-0.x\",\n rustc_semver: \"1.84.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"9fc6b43126469e3858e2fe86cafb4f0fd5068869\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.649626214Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.649650691Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.649674417Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} {"msg":"Crucible 8a28187a-87e8-4a6f-b601-619f5c589891 has session id: d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.650280644Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Upstairs opts: Upstairs UUID: 8a28187a-87e8-4a6f-b601-619f5c589891, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.650323668Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Crucible stats registered with UUID: 8a28187a-87e8-4a6f-b601-619f5c589891","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.650351685Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681574515Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.68161574Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} {"msg":"8a28187a-87e8-4a6f-b601-619f5c589891 active request set","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681691499Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681723745Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681755911Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681774308Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681791006Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681858397Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681889092Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681919088Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.681950314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.68198048Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42893)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682010556Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42235)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682041262Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59534)","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682138338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682794939Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682836163Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.68285873Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682873338Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682887416Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.682900914Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.683383178Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.683476526Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.683669929Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.683943082Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.683964339Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.683984956Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684144224Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684170471Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684184549Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684196577Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684213535Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[0]R flush_numbers[0..12]: [331, 320, 322, 330, 330, 331, 331, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684227663Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[0]R generation[0..12]: [197, 192, 193, 196, 196, 197, 197, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684241371Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684255079Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[1]R flush_numbers[0..12]: [331, 320, 322, 330, 330, 331, 331, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684268547Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[1]R generation[0..12]: [197, 192, 193, 196, 196, 197, 197, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684281985Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684295774Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[2]R flush_numbers[0..12]: [331, 320, 322, 330, 330, 331, 331, 309, 320, 319, 310, 306]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684309801Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[2]R generation[0..12]: [197, 192, 193, 196, 196, 197, 197, 186, 192, 192, 187, 184]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.68432314Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684336948Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Max found gen is 198","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2025-01-11T06:12:33.684356905Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684378232Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Next flush: 332","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684401029Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684415197Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"":"downstairs","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684430755Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684442683Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"0","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684461291Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"1","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684484648Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"client":"2","":"downstairs","session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684499276Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684511444Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"8a28187a-87e8-4a6f-b601-619f5c589891 is now active with session: d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.684524042Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2025-01-11T06:12:33.68454366Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974,"session_id":"d83fbcf8-dab4-4cee-8d6b-5eeb77c49ee5"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2025-01-11T06:12:33.684563857Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 344 len 32768 data: 15 16 13 14 14 14 11 11 02/30 Write block 466 len 28672 data: 10 10 9 12 10 11 13 03/30 Read block 539 len 40960 04/30 Flush 05/30 Write block 321 len 28672 data: 18 19 16 20 15 13 11 06/30 Write block 34 len 16384 data: 14 14 15 15 07/30 Write block 575 len 36864 data: 8 13 13 14 12 13 13 18 16 08/30 Read block 105 len 28672 09/30 Read block 89 len 28672 10/30 Read block 490 len 24576 11/30 Write block 548 len 8192 data: 17 15 12/30 Read block 154 len 32768 13/30 Write block 35 len 32768 data: 15 16 16 14 15 16 22 23 14/30 Read block 149 len 32768 15/30 Write block 507 len 40960 data: 10 12 12 9 10 10 13 14 12 13 16/30 Flush 17/30 Write block 296 len 20480 data: 17 19 21 21 19 18/30 Read block 111 len 8192 19/30 Read block 536 len 8192 20/30 Read block 433 len 12288 21/30 Read block 291 len 32768 22/30 Read block 276 len 4096 23/30 Read block 244 len 12288 24/30 Read block 596 len 4096 25/30 Read block 238 len 40960 26/30 Read block 572 len 28672 27/30 Write block 446 len 20480 data: 19 20 21 22 19 28/30 Write block 154 len 28672 data: 15 19 19 18 16 13 15 29/30 Read block 305 len 8192 30/30 Write block 571 len 24576 data: 7 8 10 9 9 14 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:14 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1016 Acked Write 5 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 3 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 7 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1015 1015 1015 Downstairs last five completed: 1015 1014 1013 1012 1011 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2025-01-11T06:12:33.704782562Z","hostname":"w-01JH9ZCPXCK74892DRS35HB2K7","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 331 331 331 F F F 1 020-039 198 198 198 333 333 333 F F F 2 040-059 198 198 198 333 333 333 F F F 3 060-079 196 196 196 330 330 330 F F F 4 080-099 196 196 196 330 330 330 F F F 5 100-119 197 197 197 331 331 331 F F F 6 120-139 197 197 197 331 331 331 F F F 7 140-159 186 186 186 309 309 309 T T T 8 160-179 192 192 192 320 320 320 T T T 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 192 192 192 319 319 319 T T T 15 300-319 194 194 194 326 326 326 T T T 16 320-339 198 198 198 333 333 333 F F F 17 340-359 198 198 198 332 332 332 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 196 196 196 330 330 330 F F F 22 440-459 196 196 196 330 330 330 T T T 23 460-479 198 198 198 332 332 332 F F F 24 480-499 194 194 194 328 328 328 F F F 25 500-519 198 198 198 333 333 333 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 198 198 198 333 333 333 F F F 28 560-579 198 198 198 333 333 333 T T T 29 580-599 198 198 198 333 333 333 F F F Max gen: 198, Max flush: 333 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 197 197 197 331 331 331 F F F 1 020-039 198 198 198 333 333 333 F F F 2 040-059 198 198 198 333 333 333 F F F 3 060-079 196 196 196 330 330 330 F F F 4 080-099 196 196 196 330 330 330 F F F 5 100-119 197 197 197 331 331 331 F F F 6 120-139 197 197 197 331 331 331 F F F 7 140-159 199 199 199 334 334 334 F F F 8 160-179 199 199 199 334 334 334 F F F 9 180-199 192 192 192 319 319 319 F F F 10 200-219 187 187 187 310 310 310 F F F 11 220-239 184 184 184 306 306 306 F F F 12 240-259 193 193 193 322 322 322 F F F 13 260-279 193 193 193 322 322 322 F F F 14 280-299 199 199 199 334 334 334 F F F 15 300-319 199 199 199 334 334 334 F F F 16 320-339 198 198 198 333 333 333 F F F 17 340-359 198 198 198 332 332 332 F F F 18 360-379 196 196 196 330 330 330 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 191 191 191 317 317 317 F F F 21 420-439 196 196 196 330 330 330 F F F 22 440-459 199 199 199 334 334 334 F F F 23 460-479 198 198 198 332 332 332 F F F 24 480-499 194 194 194 328 328 328 F F F 25 500-519 198 198 198 333 333 333 F F F 26 520-539 188 188 188 314 314 314 F F F 27 540-559 198 198 198 333 333 333 F F F 28 560-579 199 199 199 334 334 334 F F F 29 580-599 198 198 198 333 333 333 F F F Max gen: 199, Max flush: 334 7:20 Test duration Test completed real 7:20.613255193 user 55.350581088 sys 19.326372916 trap 0.077705428 tflt 0.088511266 dflt 0.164343629 kflt 0.006497618 lock 11:43:19.763502395 slp 35:37.685547524 lat 5.667650296 stop 0.061138522