Turn off color for downstairs dump Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.30061826 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.052524783 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.05157386 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1184 1185 1186 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.66131266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:02.715177765Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.715651278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.751761775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.752667672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.752762289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.752783138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.752799467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188} {"msg":"Crucible f9a438bb-aa60-422a-97d4-11b98719e528 has session id: b1c811c2-956d-45ce-98b1-a8ee5d2f0a60","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.753585609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Upstairs opts: Upstairs UUID: f9a438bb-aa60-422a-97d4-11b98719e528, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.753629207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Crucible stats registered with UUID: f9a438bb-aa60-422a-97d4-11b98719e528","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.753653117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.790962919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791012917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188} {"msg":"f9a438bb-aa60-422a-97d4-11b98719e528 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791102874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791122204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791200401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791233529Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791258489Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791284758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791303267Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791327746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791352515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791370305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_connection connected from Ok(127.0.0.1:42029)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791744651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"io task","client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_connection connected from Ok(127.0.0.1:48207)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79178471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"io task","client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_connection connected from Ok(127.0.0.1:46912)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.791809299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"io task","client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79317479Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.793211708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.793240407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.793264296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.793285366Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.793305565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.794503622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.794647426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.794685535Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795300443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795337681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.7953619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79538274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795404869Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795426638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795446688Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795468487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795605682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795634381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79565091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795710058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795730347Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795745507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795759586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795773546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795787725Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795800875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795814314Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795827564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795895141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"":"downstairs","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79591472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79592752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"0","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.79594427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"1","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795958249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"client":"2","":"downstairs","session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795972498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795984958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"f9a438bb-aa60-422a-97d4-11b98719e528 is now active with session: b1c811c2-956d-45ce-98b1-a8ee5d2f0a60","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.795997868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.796010097Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188,"session_id":"b1c811c2-956d-45ce-98b1-a8ee5d2f0a60"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:02.796022317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1188} Wait for a query_work_queue command to finish before sending IO Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 88 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 256 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.652936872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:10.695075942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.695457638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.726109471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.726669241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.72670659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.726720599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.726734909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} {"msg":"Crucible e66af3e0-27d0-4d5c-b49d-b041208f1bec has session id: 67675f15-1838-468e-9692-7ef897793001","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.727163163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Upstairs opts: Upstairs UUID: e66af3e0-27d0-4d5c-b49d-b041208f1bec, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.727203482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Crucible stats registered with UUID: e66af3e0-27d0-4d5c-b49d-b041208f1bec","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.727227801Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.756974327Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757023205Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} {"msg":"e66af3e0-27d0-4d5c-b49d-b041208f1bec active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757040075Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757055964Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757084263Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757102212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757119342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757141821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.75715918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.75717589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757192889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757219248Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_connection connected from Ok(127.0.0.1:33830)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757254787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"io task","client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_connection connected from Ok(127.0.0.1:57648)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757288006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"io task","client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_connection connected from Ok(127.0.0.1:49593)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.757322244Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"io task","client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758087447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758129485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758162304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758196533Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758231002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758248761Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758810231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758852779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.758977015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759434708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759465037Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759480706Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759549254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759569103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759582763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759595222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759608572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759622472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759636331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759650491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.75966408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.75967767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759696439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759710608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759724228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759738207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759751427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759764846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759778666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759791805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"":"downstairs","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759804885Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759816984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"0","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759841844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"1","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759855033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"client":"2","":"downstairs","session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759868283Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759880612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"e66af3e0-27d0-4d5c-b49d-b041208f1bec is now active with session: 67675f15-1838-468e-9692-7ef897793001","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759893322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759905361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190,"session_id":"67675f15-1838-468e-9692-7ef897793001"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:10.759920231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 423 len 4096 data: 2 02/30 Flush 03/30 Write block 514 len 36864 data: 2 2 2 2 2 2 2 2 2 04/30 Read block 14 len 32768 05/30 Read block 48 len 8192 06/30 Read block 583 len 20480 07/30 Read block 155 len 12288 08/30 Read block 174 len 40960 09/30 Write block 422 len 16384 data: 2 3 2 2 10/30 Write block 162 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Read block 308 len 12288 12/30 Read block 557 len 36864 13/30 Read block 256 len 36864 14/30 Flush 15/30 Write block 67 len 24576 data: 2 2 2 2 2 2 16/30 Write block 327 len 28672 data: 2 2 2 2 2 2 2 17/30 Read block 472 len 24576 18/30 Write block 565 len 24576 data: 2 2 2 2 2 2 19/30 Write block 46 len 40960 data: 2 2 2 2 2 2 2 2 2 2 20/30 Read block 548 len 16384 21/30 Write block 479 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Write block 113 len 32768 data: 2 2 2 2 2 2 2 2 23/30 Read block 307 len 8192 24/30 Read block 233 len 32768 25/30 Flush 26/30 Write block 565 len 28672 data: 3 3 3 3 3 3 2 27/30 Read block 403 len 24576 28/30 Write block 520 len 20480 data: 3 3 3 2 2 29/30 Write block 104 len 8192 data: 2 2 30/30 Read block 343 len 32768 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 5 Done Sent Sent false 1028 Acked Write 2 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 3 3 6 Done 5 2 2 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:10.967390691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1190} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 2 2 2 4 4 4 T T T 6 120-139 2 2 2 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 4 4 4 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 4 4 4 F F F 24 480-499 2 2 2 4 4 4 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 2 2 2 3 3 3 T T T 27 540-559 1 1 1 1 1 1 F F F 28 560-579 2 2 2 4 4 4 T T T 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 4 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F F F 2 040-059 2 2 2 4 4 4 F F F 3 060-079 2 2 2 4 4 4 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 2 2 2 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 1 1 1 1 1 1 F F F 15 300-319 1 1 1 1 1 1 F F F 16 320-339 2 2 2 4 4 4 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 4 4 4 F F F 24 480-499 2 2 2 4 4 4 F F F 25 500-519 2 2 2 3 3 3 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 3 3 3 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 5 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.276198578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.320257427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.320876345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.320915754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.320932273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.320949722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} {"msg":"Crucible 8af05db0-62eb-49e1-889a-b24e6aafea8b has session id: e8de392b-faec-4505-b8d3-4be03058a912","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.321384207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Upstairs opts: Upstairs UUID: 8af05db0-62eb-49e1-889a-b24e6aafea8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.321434825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Crucible stats registered with UUID: 8af05db0-62eb-49e1-889a-b24e6aafea8b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.321465194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351567057Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351593786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} {"msg":"8af05db0-62eb-49e1-889a-b24e6aafea8b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351649174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351684583Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351715492Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351734421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351751731Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35176846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351785429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351802129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351819358Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351845107Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_connection connected from Ok(127.0.0.1:41123)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.351894165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"io task","client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_connection connected from Ok(127.0.0.1:48769)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352101708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"io task","client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_connection connected from Ok(127.0.0.1:48594)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352133327Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"io task","client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352789033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352830402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35286443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352893289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352918508Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.352933918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.353380852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.353539556Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.353684401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.353937292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.353966321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35398138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354130615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354152614Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354166894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354222541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354243351Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 4, 4, 1, 5, 4, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35425747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[0]R generation[0..12]: [1, 1, 2, 2, 1, 3, 2, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35427138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354285849Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 4, 4, 1, 5, 4, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354299789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[1]R generation[0..12]: [1, 1, 2, 2, 1, 3, 2, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354313368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354327518Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 4, 4, 1, 5, 4, 1, 3, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354341047Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[2]R generation[0..12]: [1, 1, 2, 2, 1, 3, 2, 1, 2, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354354697Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354368796Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354381986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354397745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354411115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354424234Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354445853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354457943Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"0","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354471123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"1","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354484302Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"client":"2","":"downstairs","session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354497511Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354509821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"8af05db0-62eb-49e1-889a-b24e6aafea8b is now active with session: e8de392b-faec-4505-b8d3-4be03058a912","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.354522671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35453466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198,"session_id":"e8de392b-faec-4505-b8d3-4be03058a912"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:15.35454697Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 20 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Read block 30 len 40960 03/30 Flush 04/30 Write block 1 len 28672 data: 2 2 2 2 2 2 2 05/30 Read block 372 len 12288 06/30 Read block 96 len 28672 07/30 Read block 240 len 28672 08/30 Flush 09/30 Write block 193 len 4096 data: 2 10/30 Write block 423 len 16384 data: 4 3 3 2 11/30 Read block 124 len 40960 12/30 Write block 291 len 12288 data: 2 2 2 13/30 Read block 104 len 16384 14/30 Write block 316 len 20480 data: 2 2 2 2 2 15/30 Read block 479 len 12288 16/30 Read block 449 len 16384 17/30 Read block 319 len 36864 18/30 Write block 499 len 16384 data: 2 2 2 2 19/30 Read block 121 len 4096 20/30 Flush 21/30 Write block 56 len 32768 data: 2 2 2 2 2 2 2 2 22/30 Read block 550 len 4096 23/30 Write block 332 len 20480 data: 3 3 2 2 2 24/30 Flush 25/30 Write block 505 len 40960 data: 2 2 2 2 2 2 2 2 2 3 26/30 Read block 497 len 8192 27/30 Read block 218 len 4096 28/30 Write block 468 len 8192 data: 2 2 29/30 Write block 213 len 40960 data: 2 2 2 2 2 2 2 2 2 2 30/30 Read block 122 len 4096 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 10 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 10 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 2 4 Done 6 4 4 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:15.37864177Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1198} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 6 6 6 F F F 2 040-059 4 4 4 9 9 9 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 2 2 2 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 4 4 4 8 8 8 F F F 10 200-219 1 1 1 1 1 1 T T T 11 220-239 1 1 1 1 1 1 T T T 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 2 2 2 4 4 4 T T T 24 480-499 4 4 4 8 8 8 F F F 25 500-519 4 4 4 8 8 8 T T T 26 520-539 3 3 3 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 3 3 3 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 9 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 4 4 4 6 6 6 F F F 2 040-059 4 4 4 9 9 9 F F F 3 060-079 4 4 4 9 9 9 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 2 2 2 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 2 2 2 3 3 3 F F F 9 180-199 4 4 4 8 8 8 F F F 10 200-219 5 5 5 10 10 10 F F F 11 220-239 5 5 5 10 10 10 F F F 12 240-259 1 1 1 1 1 1 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 4 4 4 8 8 8 F F F 16 320-339 4 4 4 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 1 1 1 1 1 F F F 20 400-419 1 1 1 1 1 1 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 5 5 5 10 10 10 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 3 3 3 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 5, Max flush: 10 Begin loop 3 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.661640288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.704941995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.705559903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.705605091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.705623181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.7056413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} {"msg":"Crucible 0e6acb5a-3fd2-4f90-9405-319e7772b0da has session id: 472839b3-8162-4e5a-92eb-23e402f63e06","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.706018326Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Upstairs opts: Upstairs UUID: 0e6acb5a-3fd2-4f90-9405-319e7772b0da, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.706060065Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Crucible stats registered with UUID: 0e6acb5a-3fd2-4f90-9405-319e7772b0da","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.706079574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.735775362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.735819771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} {"msg":"0e6acb5a-3fd2-4f90-9405-319e7772b0da active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.73585128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.735871409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.735915027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.735952716Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.735982585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736000894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736018303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736048622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736082911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.73611975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_connection connected from Ok(127.0.0.1:54442)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736175748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_connection connected from Ok(127.0.0.1:53077)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736218666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_connection connected from Ok(127.0.0.1:39573)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.736265934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737020737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737051746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737075025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737103374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737130793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737157782Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737642135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737807779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.737835388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738176705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738216464Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738244743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738392108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738420297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738436296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738455666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738469815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[0]R flush_numbers[0..12]: [7, 6, 9, 9, 1, 5, 4, 1, 3, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738510364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[0]R generation[0..12]: [4, 4, 4, 4, 1, 3, 2, 1, 2, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738531423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738546462Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[1]R flush_numbers[0..12]: [7, 6, 9, 9, 1, 5, 4, 1, 3, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738560102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[1]R generation[0..12]: [4, 4, 4, 4, 1, 3, 2, 1, 2, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738573871Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738588011Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[2]R flush_numbers[0..12]: [7, 6, 9, 9, 1, 5, 4, 1, 3, 8, 10, 10]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.7386015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[2]R generation[0..12]: [4, 4, 4, 4, 1, 3, 2, 1, 2, 4, 5, 5]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738623929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738652788Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738688847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Generation requested: 6 >= found:6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738718496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Next flush: 11","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738745475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738773774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"":"downstairs","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738793193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738805863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"0","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738829662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"1","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738857091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"client":"2","":"downstairs","session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.73888553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738900459Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"0e6acb5a-3fd2-4f90-9405-319e7772b0da is now active with session: 472839b3-8162-4e5a-92eb-23e402f63e06","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738915319Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738937688Z","hostnameRegion: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206,"session_id":"472839b3-8162-4e5a-92eb-23e402f63e06"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:19.738964217Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 31 len 4096 data: 2 02/30 Read block 298 len 8192 03/30 Read block 417 len 32768 04/30 Flush 05/30 Write block 69 len 12288 data: 3 3 3 06/30 Write block 185 len 40960 data: 2 2 2 2 2 2 2 2 3 2 07/30 Write block 239 len 8192 data: 2 2 08/30 Read block 211 len 36864 09/30 Read block 227 len 12288 10/30 Write block 162 len 4096 data: 3 11/30 Read block 370 len 16384 12/30 Write block 305 len 32768 data: 2 2 2 2 2 2 2 2 13/30 Read block 281 len 32768 14/30 Read block 563 len 8192 15/30 Read block 477 len 36864 16/30 Read block 396 len 40960 17/30 Read block 287 len 4096 18/30 Read block 87 len 8192 19/30 Flush 20/30 Write block 40 len 16384 data: 2 2 2 2 21/30 Read block 466 len 32768 22/30 Write block 227 len 20480 data: 2 2 2 2 2 23/30 Write block 394 len 40960 data: 2 2 2 2 2 2 2 2 2 2 24/30 Flush 25/30 Flush 26/30 Write block 368 len 32768 data: 2 2 2 2 2 2 2 2 27/30 Read block 345 len 8192 28/30 Write block 409 len 8192 data: 2 2 29/30 Write block 218 len 36864 data: 3 3 3 3 3 2 2 2 2 30/30 Write block 574 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:19.761361249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1206} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 13 13 13 F F F 3 060-079 6 6 6 12 12 12 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 2 2 2 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 12 12 12 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 5 5 5 10 10 10 T T T 11 220-239 6 6 6 13 13 13 T T T 12 240-259 6 6 6 12 12 12 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 4 4 4 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 T T T 19 380-399 6 6 6 13 13 13 F F F 20 400-419 6 6 6 13 13 13 T T T 21 420-439 4 4 4 8 8 8 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 5 5 5 10 10 10 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 3 3 3 5 5 5 T T T 29 580-599 1 1 1 1 1 1 T T T Max gen: 6, Max flush: 13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 6 6 6 13 13 13 F F F 3 060-079 6 6 6 12 12 12 F F F 4 080-099 1 1 1 1 1 1 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 2 2 2 4 4 4 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 6 6 6 12 12 12 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 7 7 7 14 14 14 F F F 12 240-259 6 6 6 12 12 12 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 4 4 4 9 9 9 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 7 7 7 14 14 14 F F F 19 380-399 6 6 6 13 13 13 F F F 20 400-419 7 7 7 14 14 14 F F F 21 420-439 4 4 4 8 8 8 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 5 5 5 10 10 10 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 F F F 26 520-539 3 3 3 5 5 5 F F F 27 540-559 1 1 1 1 1 1 F F F 28 560-579 7 7 7 14 14 14 F F F 29 580-599 7 7 7 14 14 14 F F F Max gen: 7, Max flush: 14 Begin loop 4 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.078300042Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.122064582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.122705079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.122747378Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.122764797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.122783506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} {"msg":"Crucible aad884a8-edb7-4ef1-afeb-73068101a9cd has session id: 24ee046e-1a60-486c-b00e-3f04d464d693","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.12322438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Upstairs opts: Upstairs UUID: aad884a8-edb7-4ef1-afeb-73068101a9cd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.123267019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Crucible stats registered with UUID: aad884a8-edb7-4ef1-afeb-73068101a9cd","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.123296878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153394231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.1534216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} {"msg":"aad884a8-edb7-4ef1-afeb-73068101a9cd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153480088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153515877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153557025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153592514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153627023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153664202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15370026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153733749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153767958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153914022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_connection connected from Ok(127.0.0.1:63467)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.153952291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"io task","client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_connection connected from Ok(127.0.0.1:63770)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15398808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"io task","client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_connection connected from Ok(127.0.0.1:39620)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154021109Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"io task","client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154379936Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154410804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154558819Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154580608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154596438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.154622637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155041182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15507648Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155340851Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15562692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15565349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155668399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155681449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155695238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155709337Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15591576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155944189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[0]R flush_numbers[0..12]: [7, 11, 13, 12, 1, 5, 4, 1, 12, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155959559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[0]R generation[0..12]: [4, 6, 6, 6, 1, 3, 2, 1, 6, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155973698Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.155988558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[1]R flush_numbers[0..12]: [7, 11, 13, 12, 1, 5, 4, 1, 12, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156018737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[1]R generation[0..12]: [4, 6, 6, 6, 1, 3, 2, 1, 6, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156033996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156052095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[2]R flush_numbers[0..12]: [7, 11, 13, 12, 1, 5, 4, 1, 12, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156066305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[2]R generation[0..12]: [4, 6, 6, 6, 1, 3, 2, 1, 6, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156079784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156093814Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Max found gen is 8","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156107133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156120353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Next flush: 15","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156133432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156157641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"":"downstairs","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156171101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156183271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"0","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.15619671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"1","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156210179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"client":"2","":"downstairs","session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156223469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156235669Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"aad884a8-edb7-4ef1-afeb-73068101a9cd is now active with session: 24ee046e-1a60-486c-b00e-3f04d464d693","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156248598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156260508Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214,"session_id":"24ee046e-1a60-486c-b00e-3f04d464d693"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:24.156274497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 143 len 24576 data: 2 2 2 2 2 2 02/30 Flush 03/30 Write block 155 len 28672 data: 2 2 2 2 2 2 2 04/30 Read block 443 len 12288 05/30 Flush 06/30 Write block 81 len 40960 data: 2 2 2 2 2 2 2 2 2 2 07/30 Write block 546 len 36864 data: 2 2 2 2 2 2 2 2 2 08/30 Write block 339 len 24576 data: 2 2 2 2 2 2 09/30 Write block 567 len 4096 data: 4 10/30 Write block 380 len 24576 data: 2 2 2 2 2 2 11/30 Write block 340 len 12288 data: 3 3 3 12/30 Write block 331 len 24576 data: 3 4 4 3 3 3 13/30 Read block 592 len 32768 14/30 Write block 47 len 32768 data: 3 3 3 3 3 3 3 3 15/30 Read block 574 len 24576 16/30 Flush 17/30 Write block 152 len 20480 data: 2 2 2 3 3 18/30 Read block 243 len 20480 19/30 Read block 12 len 12288 20/30 Write block 523 len 40960 data: 3 3 2 2 2 2 2 2 2 2 21/30 Read block 408 len 4096 22/30 Read block 156 len 16384 23/30 Read block 220 len 8192 24/30 Flush 25/30 Write block 438 len 16384 data: 2 2 2 2 26/30 Write block 248 len 24576 data: 2 2 2 2 2 2 27/30 Write block 145 len 12288 data: 3 3 3 28/30 Read block 96 len 8192 29/30 Write block 134 len 16384 data: 2 2 2 2 30/30 Write block 500 len 28672 data: 3 3 3 2 2 3 3 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:24.182394414Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1214} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 6 6 6 12 12 12 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 2 2 2 4 4 4 T T T 7 140-159 8 8 8 18 18 18 T T T 8 160-179 8 8 8 16 16 16 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 7 7 7 14 14 14 F F F 12 240-259 6 6 6 12 12 12 T T T 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 8 8 8 17 17 17 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 7 7 7 14 14 14 F F F 19 380-399 8 8 8 17 17 17 F F F 20 400-419 7 7 7 14 14 14 F F F 21 420-439 4 4 4 8 8 8 T T T 22 440-459 1 1 1 1 1 1 T T T 23 460-479 5 5 5 10 10 10 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 5 5 5 10 10 10 T T T 26 520-539 8 8 8 18 18 18 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 8 8 8 17 17 17 F F F 29 580-599 7 7 7 14 14 14 F F F Max gen: 8, Max flush: 18 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 6 6 6 11 11 11 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 6 6 6 12 12 12 F F F 4 080-099 8 8 8 17 17 17 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 9 9 9 19 19 19 F F F 7 140-159 9 9 9 19 19 19 F F F 8 160-179 8 8 8 16 16 16 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 7 7 7 14 14 14 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 1 1 1 1 1 1 F F F 14 280-299 4 4 4 8 8 8 F F F 15 300-319 6 6 6 12 12 12 F F F 16 320-339 8 8 8 17 17 17 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 7 7 7 14 14 14 F F F 19 380-399 8 8 8 17 17 17 F F F 20 400-419 7 7 7 14 14 14 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 5 5 5 10 10 10 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 8 8 8 18 18 18 F F F 27 540-559 8 8 8 17 17 17 F F F 28 560-579 8 8 8 17 17 17 F F F 29 580-599 7 7 7 14 14 14 F F F Max gen: 9, Max flush: 19 Begin loop 5 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.50813804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:28.551057931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.551459086Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.582962889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.583507449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.583543128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.583561128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.583575767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} {"msg":"Crucible 25f6d800-bcb3-407b-bb91-8dc2c40bdc8b has session id: e702b40d-bb5f-462b-a31d-a2cc12c10c8f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.584018431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Upstairs opts: Upstairs UUID: 25f6d800-bcb3-407b-bb91-8dc2c40bdc8b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.58405957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Crucible stats registered with UUID: 25f6d800-bcb3-407b-bb91-8dc2c40bdc8b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.584094918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613735328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613771137Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} {"msg":"25f6d800-bcb3-407b-bb91-8dc2c40bdc8b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613802506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613820525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613847494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613876293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613910882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613946531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.613981649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614018918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614060396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614094455Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:44397)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614128374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"io task","client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:53976)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614159103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"io task","client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45560)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.61424967Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"io task","client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614844828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614887637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614920685Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614950944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.614981473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.61506848Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.615513164Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.615644589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.615743656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616110142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616144191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616160041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.61618081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616195069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616208429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616278816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616299186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[0]R flush_numbers[0..12]: [7, 11, 17, 12, 17, 5, 19, 19, 16, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616313305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[0]R generation[0..12]: [4, 6, 8, 6, 8, 3, 9, 9, 8, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616327425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616341784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[1]R flush_numbers[0..12]: [7, 11, 17, 12, 17, 5, 19, 19, 16, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616355713Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[1]R generation[0..12]: [4, 6, 8, 6, 8, 3, 9, 9, 8, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616369223Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616383372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[2]R flush_numbers[0..12]: [7, 11, 17, 12, 17, 5, 19, 19, 16, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616397142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[2]R generation[0..12]: [4, 6, 8, 6, 8, 3, 9, 9, 8, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616410652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616427481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616440961Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.61645434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616467569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616480559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"":"downstairs","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"No downstairs reconciliation required","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616502618Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616514888Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"0","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616528567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"1","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616542297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"client":"2","":"downstairs","session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616556126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616568586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"25f6d800-bcb3-407b-bb91-8dc2c40bdc8b is now active with session: e702b40d-bb5f-462b-a31d-a2cc12c10c8f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616581596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616593785Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222,"session_id":"e702b40d-bb5f-462b-a31d-a2cc12c10c8f"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:28.616606195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 471 len 8192 data: 2 2 02/30 Flush 03/30 Write block 32 len 32768 data: 2 2 2 2 2 2 2 2 04/30 Read block 324 len 16384 05/30 Write block 74 len 12288 data: 2 2 2 06/30 Write block 562 len 12288 data: 2 2 2 07/30 Read block 168 len 4096 08/30 Read block 450 len 12288 09/30 Write block 418 len 4096 data: 2 10/30 Write block 380 len 4096 data: 3 11/30 Read block 442 len 4096 12/30 Write block 594 len 4096 data: 2 13/30 Read block 303 len 16384 14/30 Read block 194 len 32768 15/30 Write block 337 len 8192 data: 2 2 16/30 Read block 426 len 4096 17/30 Write block 274 len 12288 data: 2 2 2 18/30 Write block 388 len 16384 data: 2 2 2 2 19/30 Read block 333 len 40960 20/30 Flush 21/30 Write block 286 len 32768 data: 2 2 2 2 2 3 3 3 22/30 Read block 555 len 40960 23/30 Write block 546 len 24576 data: 3 3 3 3 3 3 24/30 Flush 25/30 Write block 305 len 8192 data: 3 3 26/30 Write block 282 len 28672 data: 2 2 2 2 3 3 3 27/30 Read block 552 len 12288 28/30 Write block 574 len 4096 data: 3 29/30 Write block 75 len 24576 data: 3 3 2 2 2 2 30/30 Write block 157 len 16384 data: 3 3 3 3 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 2 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:28.63806496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1222} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 10 10 10 21 21 21 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 10 10 10 21 21 21 T T T 4 080-099 8 8 8 17 17 17 T T T 5 100-119 3 3 3 5 5 5 F F F 6 120-139 9 9 9 19 19 19 F F F 7 140-159 9 9 9 19 19 19 T T T 8 160-179 8 8 8 16 16 16 T T T 9 180-199 6 6 6 12 12 12 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 7 7 7 14 14 14 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 10 10 10 21 21 21 F F F 14 280-299 10 10 10 22 22 22 T T T 15 300-319 6 6 6 12 12 12 T T T 16 320-339 10 10 10 21 21 21 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 7 7 7 14 14 14 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 10 10 10 21 21 21 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 8 8 8 18 18 18 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 10 10 10 21 21 21 T T T 29 580-599 10 10 10 21 21 21 F F F Max gen: 10, Max flush: 22 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 10 10 10 21 21 21 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 11 11 11 23 23 23 F F F 4 080-099 11 11 11 23 23 23 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 9 9 9 19 19 19 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 11 11 11 23 23 23 F F F 9 180-199 6 6 6 12 12 12 F F F 10 200-219 7 7 7 14 14 14 F F F 11 220-239 7 7 7 14 14 14 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 10 10 10 21 21 21 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 11 11 11 23 23 23 F F F 16 320-339 10 10 10 21 21 21 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 7 7 7 14 14 14 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 10 10 10 21 21 21 F F F 21 420-439 9 9 9 19 19 19 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 9 9 9 19 19 19 F F F 26 520-539 8 8 8 18 18 18 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 11 11 11 23 23 23 F F F 29 580-599 10 10 10 21 21 21 F F F Max gen: 11, Max flush: 23 Begin loop 6 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.007388182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:33.049439044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.04984052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.080669197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.081295515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.081331123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.081345273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.081359962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} {"msg":"Crucible a5c381f1-1454-4f7e-baa4-fd69eda7b822 has session id: 330c589e-7663-441e-af69-a3b5727185de","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.081938691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Upstairs opts: Upstairs UUID: a5c381f1-1454-4f7e-baa4-fd69eda7b822, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.081997809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Crucible stats registered with UUID: a5c381f1-1454-4f7e-baa4-fd69eda7b822","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.082031698Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112061724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112096543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} {"msg":"a5c381f1-1454-4f7e-baa4-fd69eda7b822 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112116222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112132352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.11215952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.11217724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112194029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112207289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112221578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112245527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112271027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112295715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_connection connected from Ok(127.0.0.1:61223)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112337234Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"io task","client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_connection connected from Ok(127.0.0.1:55897)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112485469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"io task","client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_connection connected from Ok(127.0.0.1:52711)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.112574465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"io task","client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113149005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113178534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113194233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113208673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113222322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.11328418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113862959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113904257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.113934656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114455958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114486227Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114502336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114515315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114532665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114546574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114558904Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114572473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[0]R flush_numbers[0..12]: [7, 21, 17, 23, 23, 5, 19, 23, 23, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114586333Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[0]R generation[0..12]: [4, 10, 8, 11, 11, 3, 9, 11, 11, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114600132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114614392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[1]R flush_numbers[0..12]: [7, 21, 17, 23, 23, 5, 19, 23, 23, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114628111Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[1]R generation[0..12]: [4, 10, 8, 11, 11, 3, 9, 11, 11, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114641601Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.11465593Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[2]R flush_numbers[0..12]: [7, 21, 17, 23, 23, 5, 19, 23, 23, 12, 14, 14]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.1146702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[2]R generation[0..12]: [4, 10, 8, 11, 11, 3, 9, 11, 11, 6, 7, 7]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114683799Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114697969Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114711298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114726658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Next flush: 24","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114739897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114753057Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"":"downstairs","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114774366Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114786646Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"0","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114800055Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"1","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114813325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"client":"2","":"downstairs","session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114826934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114839474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"a5c381f1-1454-4f7e-baa4-fd69eda7b822 is now active with session: 330c589e-7663-441e-af69-a3b5727185de","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114852523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114864633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230,"session_id":"330c589e-7663-441e-af69-a3b5727185de"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:33.114876852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 370 len 16384 data: 3 3 3 3 02/30 Write block 134 len 20480 data: 3 3 3 3 2 03/30 Write block 524 len 12288 data: 4 3 3 04/30 Read block 253 len 40960 05/30 Write block 170 len 28672 data: 2 2 2 2 2 2 2 06/30 Flush 07/30 Write block 189 len 4096 data: 3 08/30 Write block 213 len 36864 data: 3 3 3 3 3 4 4 4 4 09/30 Read block 404 len 28672 10/30 Read block 167 len 28672 11/30 Read block 116 len 16384 12/30 Write block 22 len 28672 data: 3 3 3 3 3 3 3 13/30 Read block 104 len 16384 14/30 Read block 334 len 24576 15/30 Read block 408 len 12288 16/30 Write block 270 len 32768 data: 2 2 2 2 3 3 3 2 17/30 Read block 353 len 4096 18/30 Read block 244 len 12288 19/30 Read block 358 len 16384 20/30 Read block 51 len 12288 21/30 Write block 93 len 12288 data: 2 2 2 22/30 Read block 489 len 8192 23/30 Flush 24/30 Write block 420 len 8192 data: 2 2 25/30 Write block 316 len 32768 data: 3 3 3 3 3 2 2 2 26/30 Write block 308 len 28672 data: 3 3 3 3 3 2 2 27/30 Write block 510 len 16384 data: 3 3 3 3 28/30 Read block 443 len 20480 29/30 Write block 168 len 12288 data: 3 3 3 30/30 Write block 587 len 28672 data: 2 2 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:33.136051088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1230} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 12 12 12 25 25 25 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 11 11 11 23 23 23 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 12 12 12 24 24 24 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 12 12 12 24 24 24 T T T 9 180-199 12 12 12 25 25 25 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 12 12 12 25 25 25 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 11 11 11 23 23 23 T T T 16 320-339 10 10 10 21 21 21 T T T 17 340-359 8 8 8 17 17 17 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 10 10 10 21 21 21 F F F 21 420-439 9 9 9 19 19 19 T T T 22 440-459 9 9 9 19 19 19 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 9 9 9 19 19 19 T T T 26 520-539 12 12 12 24 24 24 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 11 11 11 23 23 23 F F F 29 580-599 10 10 10 21 21 21 T T T Max gen: 12, Max flush: 25 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 12 12 12 25 25 25 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 11 11 11 23 23 23 F F F 4 080-099 12 12 12 25 25 25 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 12 12 12 24 24 24 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 13 13 13 26 26 26 F F F 9 180-199 12 12 12 25 25 25 F F F 10 200-219 12 12 12 25 25 25 F F F 11 220-239 12 12 12 25 25 25 F F F 12 240-259 9 9 9 19 19 19 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 13 13 13 26 26 26 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 10 10 10 21 21 21 F F F 21 420-439 13 13 13 26 26 26 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 13 13 13 26 26 26 F F F 26 520-539 12 12 12 24 24 24 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 11 11 11 23 23 23 F F F 29 580-599 13 13 13 26 26 26 F F F Max gen: 13, Max flush: 26 Begin loop 7 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.50790467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.551378101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.55195603Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.551992939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.552008528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.552022817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} {"msg":"Crucible 553a396f-7845-4923-92ff-39a08d48bcbf has session id: 95f7ad87-6b11-4dfb-bf17-70f64e61874e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.55249314Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Upstairs opts: Upstairs UUID: 553a396f-7845-4923-92ff-39a08d48bcbf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.552535569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Crucible stats registered with UUID: 553a396f-7845-4923-92ff-39a08d48bcbf","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.552560988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582490918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582534986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} {"msg":"553a396f-7845-4923-92ff-39a08d48bcbf active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582570375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582590204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582633372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582670751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.58270767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582743498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582858024Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.582895203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583032588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583062927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64698)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583090036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"io task","client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_connection connected from Ok(127.0.0.1:62990)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583108795Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"io task","client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50707)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583123325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"io task","client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583587768Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583616077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583634346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583648406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583668505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.583688984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.584288763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.584457536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.584478756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.584878311Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.58491746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.584935079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585067845Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585098843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585114953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585127952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585141662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[0]R flush_numbers[0..12]: [7, 25, 17, 23, 25, 5, 24, 23, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585159161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[0]R generation[0..12]: [4, 12, 8, 11, 12, 3, 12, 11, 13, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585173601Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.58518797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[1]R flush_numbers[0..12]: [7, 25, 17, 23, 25, 5, 24, 23, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.58520204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[1]R generation[0..12]: [4, 12, 8, 11, 12, 3, 12, 11, 13, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585215939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585230089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[2]R flush_numbers[0..12]: [7, 25, 17, 23, 25, 5, 24, 23, 26, 25, 25, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585247998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[2]R generation[0..12]: [4, 12, 8, 11, 12, 3, 12, 11, 13, 12, 12, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585274877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585305036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585325725Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585360164Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Next flush: 27","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585385773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585404792Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"":"downstairs","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585430431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585456251Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"0","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.58548009Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"1","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585495569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"client":"2","":"downstairs","session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585520478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585545687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"553a396f-7845-4923-92ff-39a08d48bcbf is now active with session: 95f7ad87-6b11-4dfb-bf17-70f64e61874e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585571106Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585585176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238,"session_id":"95f7ad87-6b11-4dfb-bf17-70f64e61874e"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:37.585603695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 76 len 4096 data: 4 02/30 Read block 240 len 16384 03/30 Write block 409 len 24576 data: 3 3 2 2 2 2 04/30 Read block 125 len 16384 05/30 Write block 204 len 16384 data: 2 2 2 2 06/30 Write block 432 len 20480 data: 2 2 2 2 2 07/30 Read block 139 len 36864 08/30 Read block 521 len 28672 09/30 Read block 395 len 16384 10/30 Read block 516 len 4096 11/30 Read block 478 len 20480 12/30 Write block 125 len 40960 data: 2 2 2 2 2 2 2 2 2 4 13/30 Read block 475 len 16384 14/30 Flush 15/30 Write block 86 len 40960 data: 3 3 3 3 3 2 2 3 3 3 16/30 Write block 251 len 24576 data: 3 3 3 2 2 2 17/30 Flush 18/30 Write block 565 len 8192 data: 4 4 19/30 Flush 20/30 Write block 586 len 4096 data: 2 21/30 Flush 22/30 Write block 301 len 4096 data: 2 23/30 Read block 381 len 32768 24/30 Read block 144 len 16384 25/30 Write block 183 len 20480 data: 2 2 3 3 3 26/30 Read block 578 len 8192 27/30 Read block 308 len 20480 28/30 Read block 269 len 12288 29/30 Read block 327 len 12288 30/30 Read block 164 len 24576 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 6 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 8 9 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:37.608237928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1238} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 12 12 12 25 25 25 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 F F F 4 080-099 14 14 14 28 28 28 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 14 14 14 27 27 27 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 13 13 13 26 26 26 F F F 9 180-199 12 12 12 25 25 25 T T T 10 200-219 14 14 14 27 27 27 F F F 11 220-239 12 12 12 25 25 25 F F F 12 240-259 14 14 14 28 28 28 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 13 13 13 26 26 26 T T T 16 320-339 13 13 13 26 26 26 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 13 13 13 26 26 26 F F F 26 520-539 12 12 12 24 24 24 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 14 14 14 29 29 29 F F F 29 580-599 14 14 14 30 30 30 F F F Max gen: 14, Max flush: 30 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 4 4 4 7 7 7 F F F 1 020-039 12 12 12 25 25 25 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 F F F 4 080-099 14 14 14 28 28 28 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 14 14 14 27 27 27 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 13 13 13 26 26 26 F F F 9 180-199 15 15 15 31 31 31 F F F 10 200-219 14 14 14 27 27 27 F F F 11 220-239 12 12 12 25 25 25 F F F 12 240-259 14 14 14 28 28 28 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 11 11 11 23 23 23 F F F 15 300-319 15 15 15 31 31 31 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 10 10 10 20 20 20 F F F 24 480-499 4 4 4 8 8 8 F F F 25 500-519 13 13 13 26 26 26 F F F 26 520-539 12 12 12 24 24 24 F F F 27 540-559 10 10 10 22 22 22 F F F 28 560-579 14 14 14 29 29 29 F F F 29 580-599 14 14 14 30 30 30 F F F Max gen: 15, Max flush: 31 Begin loop 8 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.876301465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:41.920403133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.920793179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.951830039Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.95235558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.952390709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.952404588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.952418688Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} {"msg":"Crucible d6895f71-ad72-4b7e-ab62-c8155476c8de has session id: 25142441-589f-46f6-8984-1af59bc8ef21","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.952930349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Upstairs opts: Upstairs UUID: d6895f71-ad72-4b7e-ab62-c8155476c8de, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.952964838Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Crucible stats registered with UUID: d6895f71-ad72-4b7e-ab62-c8155476c8de","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.952980197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.982929616Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.982959285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} {"msg":"d6895f71-ad72-4b7e-ab62-c8155476c8de active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983039812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983058432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983084791Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98309951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98311321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983126439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983139889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983153078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983176007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983200557Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_connection connected from Ok(127.0.0.1:61008)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983362711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"io task","client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_connection connected from Ok(127.0.0.1:51058)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983503316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"io task","client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_connection connected from Ok(127.0.0.1:53570)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.983537434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"io task","client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98420269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.984231059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.984399023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.984438712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.984464801Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.984524939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.984996812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98502882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985191095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98558632Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98561477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985629799Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985642968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985656918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985670607Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985798893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985828232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[0]R flush_numbers[0..12]: [7, 25, 17, 27, 28, 5, 27, 23, 26, 31, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985844881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[0]R generation[0..12]: [4, 12, 8, 14, 14, 3, 14, 11, 13, 15, 14, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985859701Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.98587411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[1]R flush_numbers[0..12]: [7, 25, 17, 27, 28, 5, 27, 23, 26, 31, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985891729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[1]R generation[0..12]: [4, 12, 8, 14, 14, 3, 14, 11, 13, 15, 14, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985905879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985919858Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[2]R flush_numbers[0..12]: [7, 25, 17, 27, 28, 5, 27, 23, 26, 31, 27, 25]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985933458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[2]R generation[0..12]: [4, 12, 8, 14, 14, 3, 14, 11, 13, 15, 14, 12]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985947078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985961027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985974257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.985987656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Next flush: 32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986000505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:03:41.986013365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"":"downstairs","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986032264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986044374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"0","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986057654Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"1","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986070873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"client":"2","":"downstairs","session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986084202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986098132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"d6895f71-ad72-4b7e-ab62-c8155476c8de is now active with session: 25142441-589f-46f6-8984-1af59bc8ef21","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986111221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986123221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246,"session_id":"25142441-589f-46f6-8984-1af59bc8ef21"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:41.986135491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 231 len 12288 data: 3 2 2 02/30 Read block 137 len 20480 03/30 Read block 201 len 28672 04/30 Read block 259 len 8192 05/30 Flush 06/30 Write block 86 len 40960 data: 4 4 4 4 4 3 3 4 4 4 07/30 Flush 08/30 Write block 488 len 28672 data: 3 2 2 2 2 2 2 09/30 Write block 19 len 4096 data: 2 10/30 Read block 219 len 16384 11/30 Read block 497 len 36864 12/30 Read block 104 len 36864 13/30 Flush 14/30 Write block 552 len 8192 data: 3 3 15/30 Read block 587 len 16384 16/30 Read block 442 len 12288 17/30 Read block 342 len 40960 18/30 Write block 126 len 8192 data: 3 3 19/30 Write block 239 len 40960 data: 3 3 2 2 2 2 2 2 2 3 20/30 Read block 456 len 12288 21/30 Read block 367 len 20480 22/30 Read block 372 len 8192 23/30 Write block 470 len 24576 data: 2 3 3 2 2 2 24/30 Read block 312 len 24576 25/30 Write block 587 len 16384 data: 3 3 3 3 26/30 Flush 27/30 Write block 17 len 28672 data: 2 2 3 3 3 4 4 28/30 Write block 25 len 40960 data: 4 4 4 4 2 2 3 3 3 3 29/30 Read block 513 len 4096 30/30 Write block 282 len 8192 data: 3 3 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 7 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:42.012063195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1246} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 34 34 34 T T T 1 020-039 12 12 12 25 25 25 T T T 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 16 16 16 35 35 35 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 13 13 13 26 26 26 F F F 9 180-199 15 15 15 31 31 31 F F F 10 200-219 14 14 14 27 27 27 F F F 11 220-239 16 16 16 35 35 35 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 11 11 11 23 23 23 T T T 15 300-319 15 15 15 31 31 31 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 16 16 16 35 35 35 F F F 24 480-499 16 16 16 34 34 34 F F F 25 500-519 13 13 13 26 26 26 F F F 26 520-539 12 12 12 24 24 24 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 14 14 14 29 29 29 F F F 29 580-599 16 16 16 35 35 35 F F F Max gen: 16, Max flush: 35 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 17 17 17 36 36 36 F F F 1 020-039 17 17 17 36 36 36 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 3 3 3 5 5 5 F F F 6 120-139 16 16 16 35 35 35 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 13 13 13 26 26 26 F F F 9 180-199 15 15 15 31 31 31 F F F 10 200-219 14 14 14 27 27 27 F F F 11 220-239 16 16 16 35 35 35 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 17 17 17 36 36 36 F F F 15 300-319 15 15 15 31 31 31 F F F 16 320-339 13 13 13 26 26 26 F F F 17 340-359 8 8 8 17 17 17 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 F F F 20 400-419 14 14 14 27 27 27 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 16 16 16 35 35 35 F F F 24 480-499 16 16 16 34 34 34 F F F 25 500-519 13 13 13 26 26 26 F F F 26 520-539 12 12 12 24 24 24 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 14 14 14 29 29 29 F F F 29 580-599 16 16 16 35 35 35 F F F Max gen: 17, Max flush: 36 Begin loop 9 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.328375921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:46.372012366Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.372404521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.402984657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.403669553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.403707691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.403721961Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.40373667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} {"msg":"Crucible d0b53b10-7f18-4e29-9d2a-fdbddd520e50 has session id: 315cf30d-4c2d-4c87-8398-558ceb449c95","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.404280271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Upstairs opts: Upstairs UUID: d0b53b10-7f18-4e29-9d2a-fdbddd520e50, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.404321929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Crucible stats registered with UUID: d0b53b10-7f18-4e29-9d2a-fdbddd520e50","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.404346258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.433961119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.433991418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} {"msg":"d0b53b10-7f18-4e29-9d2a-fdbddd520e50 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434060116Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434095204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434125193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434145443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434163992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434181611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.43421181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434246469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434279108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.43449543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_connection connected from Ok(127.0.0.1:44865)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434521309Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"io task","client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_connection connected from Ok(127.0.0.1:60436)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434540018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"io task","client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_connection connected from Ok(127.0.0.1:59272)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.434557998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"io task","client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435154846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435184885Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435201765Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435216124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435278082Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435311021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435822572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435853281Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.435901279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436402201Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.43643119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436446249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436529637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436549276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436563115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436576165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436589754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[0]R flush_numbers[0..12]: [36, 36, 17, 27, 33, 5, 35, 23, 26, 31, 27, 35]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436603644Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[0]R generation[0..12]: [17, 17, 8, 14, 16, 3, 16, 11, 13, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436617463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436631783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[1]R flush_numbers[0..12]: [36, 36, 17, 27, 33, 5, 35, 23, 26, 31, 27, 35]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436645662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[1]R generation[0..12]: [17, 17, 8, 14, 16, 3, 16, 11, 13, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436659262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436673211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[2]R flush_numbers[0..12]: [36, 36, 17, 27, 33, 5, 35, 23, 26, 31, 27, 35]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436690031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[2]R generation[0..12]: [17, 17, 8, 14, 16, 3, 16, 11, 13, 15, 14, 16]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.43670407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.4367182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436731269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436744449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Next flush: 37","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436757498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436770478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"":"downstairs","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436792127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436805316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"0","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436818536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"1","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436831686Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"client":"2","":"downstairs","session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436845095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436857615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"d0b53b10-7f18-4e29-9d2a-fdbddd520e50 is now active with session: 315cf30d-4c2d-4c87-8398-558ceb449c95","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436870664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436882884Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254,"session_id":"315cf30d-4c2d-4c87-8398-558ceb449c95"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:46.436896743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 40960 data: 2 2 2 2 3 3 4 4 4 5 02/30 Read block 125 len 24576 03/30 Write block 169 len 16384 data: 4 4 3 3 04/30 Read block 579 len 32768 05/30 Write block 102 len 12288 data: 2 2 3 06/30 Flush 07/30 Write block 229 len 4096 data: 3 08/30 Write block 393 len 28672 data: 2 3 3 3 3 3 3 09/30 Write block 330 len 36864 data: 3 4 5 5 4 4 4 3 3 10/30 Read block 366 len 16384 11/30 Write block 162 len 16384 data: 4 3 3 3 12/30 Write block 471 len 24576 data: 4 4 3 3 3 2 13/30 Write block 496 len 40960 data: 2 2 2 3 4 4 4 3 3 4 14/30 Read block 482 len 8192 15/30 Read block 64 len 24576 16/30 Write block 336 len 12288 data: 5 4 4 17/30 Read block 411 len 8192 18/30 Read block 251 len 32768 19/30 Read block 33 len 4096 20/30 Read block 385 len 24576 21/30 Read block 116 len 8192 22/30 Write block 392 len 36864 data: 2 3 4 4 4 4 4 4 3 23/30 Read block 392 len 8192 24/30 Read block 117 len 24576 25/30 Write block 334 len 36864 data: 5 5 6 5 5 3 4 4 4 26/30 Write block 408 len 4096 data: 2 27/30 Read block 288 len 16384 28/30 Write block 565 len 36864 data: 5 5 5 4 4 4 3 2 2 29/30 Write block 354 len 4096 data: 2 30/30 Read block 182 len 4096 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Write 7 Done Done Done false 1008 Acked Write 9 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 6 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 2 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 1 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 9 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:46.459762108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1254} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 37 37 37 F F F 1 020-039 18 18 18 37 37 37 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 18 18 18 37 37 37 F F F 6 120-139 16 16 16 35 35 35 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 18 18 18 37 37 37 T T T 9 180-199 15 15 15 31 31 31 F F F 10 200-219 14 14 14 27 27 27 F F F 11 220-239 16 16 16 35 35 35 T T T 12 240-259 16 16 16 35 35 35 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 17 17 17 36 36 36 F F F 15 300-319 15 15 15 31 31 31 F F F 16 320-339 13 13 13 26 26 26 T T T 17 340-359 8 8 8 17 17 17 T T T 18 360-379 12 12 12 24 24 24 F F F 19 380-399 10 10 10 21 21 21 T T T 20 400-419 14 14 14 27 27 27 T T T 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 16 16 16 35 35 35 T T T 24 480-499 16 16 16 34 34 34 T T T 25 500-519 13 13 13 26 26 26 T T T 26 520-539 12 12 12 24 24 24 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 14 14 14 29 29 29 T T T 29 580-599 16 16 16 35 35 35 F F F Max gen: 18, Max flush: 37 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 37 37 37 F F F 1 020-039 18 18 18 37 37 37 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 18 18 18 37 37 37 F F F 6 120-139 16 16 16 35 35 35 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 19 19 19 38 38 38 F F F 9 180-199 15 15 15 31 31 31 F F F 10 200-219 14 14 14 27 27 27 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 12 12 12 25 25 25 F F F 14 280-299 17 17 17 36 36 36 F F F 15 300-319 15 15 15 31 31 31 F F F 16 320-339 19 19 19 38 38 38 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 19 19 19 38 38 38 F F F 24 480-499 19 19 19 38 38 38 F F F 25 500-519 19 19 19 38 38 38 F F F 26 520-539 12 12 12 24 24 24 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 16 16 16 35 35 35 F F F Max gen: 19, Max flush: 38 Begin loop 10 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.86542136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:50.907278218Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.907653595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.938082326Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.938639077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.938683625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.938697264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.938711524Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} {"msg":"Crucible 83f8d8c1-c176-4212-ab6e-b22a0aac2c8e has session id: 97e45ba6-fdab-4964-984e-18702e63cfa2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.939140718Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Upstairs opts: Upstairs UUID: 83f8d8c1-c176-4212-ab6e-b22a0aac2c8e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.939181867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Crucible stats registered with UUID: 83f8d8c1-c176-4212-ab6e-b22a0aac2c8e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.939205706Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969426445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969465094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} {"msg":"83f8d8c1-c176-4212-ab6e-b22a0aac2c8e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969508672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969531291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.96957674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969614328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969651707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969691456Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969722224Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969751473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969781262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57143)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969811821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"io task","client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.96983678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:50630)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969865289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"io task","client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_connection connected from Ok(127.0.0.1:52331)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.969894828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"io task","client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.970576304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.970605203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.970620542Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.970635091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.970648881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.970662801Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.971302737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.971348726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.971383784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.971952294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.971981163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.971996162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972009232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972023161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972036651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.97204904Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.97206266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[0]R flush_numbers[0..12]: [37, 37, 17, 27, 33, 37, 35, 23, 38, 31, 27, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.97207659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[0]R generation[0..12]: [18, 18, 8, 14, 16, 18, 16, 11, 19, 15, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972090599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972104938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[1]R flush_numbers[0..12]: [37, 37, 17, 27, 33, 37, 35, 23, 38, 31, 27, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972118778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[1]R generation[0..12]: [18, 18, 8, 14, 16, 18, 16, 11, 19, 15, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972132268Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972149177Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[2]R flush_numbers[0..12]: [37, 37, 17, 27, 33, 37, 35, 23, 38, 31, 27, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972163096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[2]R generation[0..12]: [18, 18, 8, 14, 16, 18, 16, 11, 19, 15, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972176636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972190725Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972203915Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972217074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Next flush: 39","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972229964Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972242824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"":"downstairs","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {Wait for a query_work_queue command to finish before sending IO "msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972267083Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972279472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"0","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972292982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"1","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972306341Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"client":"2","":"downstairs","session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972319591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.97233192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"83f8d8c1-c176-4212-ab6e-b22a0aac2c8e is now active with session: 97e45ba6-fdab-4964-984e-18702e63cfa2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.97234498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972358929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262,"session_id":"97e45ba6-fdab-4964-984e-18702e63cfa2"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:50.972371269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 269 len 24576 data: 2 3 3 3 3 4 02/30 Write block 104 len 8192 data: 4 3 03/30 Write block 470 len 24576 data: 3 5 5 4 4 4 04/30 Write block 307 len 20480 data: 3 4 4 4 4 05/30 Flush 06/30 Flush 07/30 Write block 109 len 24576 data: 2 2 2 2 3 3 08/30 Write block 34 len 24576 data: 4 3 3 3 3 3 09/30 Write block 583 len 32768 data: 2 2 2 3 4 4 4 4 10/30 Read block 526 len 8192 11/30 Read block 547 len 16384 12/30 Read block 278 len 32768 13/30 Read block 306 len 32768 14/30 Read block 339 len 12288 15/30 Write block 583 len 36864 data: 3 3 3 4 5 5 5 5 3 16/30 Read block 393 len 36864 17/30 Write block 521 len 40960 data: 4 4 4 5 4 4 3 3 3 3 18/30 Flush 19/30 Write block 62 len 24576 data: 3 3 2 2 2 3 20/30 Read block 163 len 20480 21/30 Write block 9 len 4096 data: 2 22/30 Read block 378 len 16384 23/30 Read block 50 len 28672 24/30 Read block 590 len 36864 25/30 Write block 121 len 8192 data: 2 2 26/30 Write block 267 len 36864 data: 2 2 3 4 4 4 4 5 4 27/30 Write block 179 len 20480 data: 2 2 2 2 3 28/30 Write block 124 len 20480 data: 2 3 4 4 3 29/30 Read block 536 len 12288 30/30 Write block 503 len 36864 data: 4 4 5 4 3 3 3 4 4 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 6 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 4 6 Done 11 11 8 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:50.996676901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1262} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 18 18 18 37 37 37 T T T 1 020-039 20 20 20 41 41 41 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 14 14 14 27 27 27 T T T 4 080-099 16 16 16 33 33 33 F F F 5 100-119 20 20 20 41 41 41 F F F 6 120-139 16 16 16 35 35 35 T T T 7 140-159 11 11 11 23 23 23 F F F 8 160-179 19 19 19 38 38 38 T T T 9 180-199 15 15 15 31 31 31 T T T 10 200-219 14 14 14 27 27 27 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 20 20 20 39 39 39 T T T 14 280-299 17 17 17 36 36 36 F F F 15 300-319 20 20 20 39 39 39 F F F 16 320-339 19 19 19 38 38 38 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 19 19 19 38 38 38 F F F 25 500-519 19 19 19 38 38 38 T T T 26 520-539 20 20 20 41 41 41 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 20 20 20 41 41 41 F F F Max gen: 20, Max flush: 41 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 21 21 21 42 42 42 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 20 20 20 41 41 41 F F F 6 120-139 21 21 21 42 42 42 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 21 21 21 42 42 42 F F F 9 180-199 21 21 21 42 42 42 F F F 10 200-219 14 14 14 27 27 27 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 21 21 21 42 42 42 F F F 14 280-299 17 17 17 36 36 36 F F F 15 300-319 20 20 20 39 39 39 F F F 16 320-339 19 19 19 38 38 38 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 12 12 12 24 24 24 F F F 19 380-399 19 19 19 38 38 38 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 F F F 23 460-479 20 20 20 39 39 39 F F F 24 480-499 19 19 19 38 38 38 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 20 20 20 41 41 41 F F F Max gen: 21, Max flush: 42 Begin loop 11 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.364481158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:55.406589988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.406971104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.43783996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.438482827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.438530195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.438548204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.438566734Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} {"msg":"Crucible 7c531171-b196-4ecc-ad1f-e9d128ebe4e2 has session id: 08549845-9160-4e2b-bca7-e3e0638531f0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.439008058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Upstairs opts: Upstairs UUID: 7c531171-b196-4ecc-ad1f-e9d128ebe4e2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.439039727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Crucible stats registered with UUID: 7c531171-b196-4ecc-ad1f-e9d128ebe4e2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.439055326Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468534382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468570231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} {"msg":"7c531171-b196-4ecc-ad1f-e9d128ebe4e2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468623299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468645238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468672907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468690906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468715255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468737105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468758904Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468776063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468797123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468828971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58166)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"io task","client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:55589)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.468897739Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"io task","client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_connection connected from Ok(127.0.0.1:50469)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469113901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"io task","client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469618093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469644732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469659401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469673031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469786267Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.469851914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.470410804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.470442823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.470464222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471018862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471048421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471063501Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.47107651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.47109028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471103669Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471115909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471129308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[0]R flush_numbers[0..12]: [42, 41, 17, 42, 33, 41, 42, 23, 42, 42, 27, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471143158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[0]R generation[0..12]: [21, 20, 8, 21, 16, 20, 21, 11, 21, 21, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471157207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471171627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[1]R flush_numbers[0..12]: [42, 41, 17, 42, 33, 41, 42, 23, 42, 42, 27, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471185436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[1]R generation[0..12]: [21, 20, 8, 21, 16, 20, 21, 11, 21, 21, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471199266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471213245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[2]R flush_numbers[0..12]: [42, 41, 17, 42, 33, 41, 42, 23, 42, 42, 27, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471230365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[2]R generation[0..12]: [21, 20, 8, 21, 16, 20, 21, 11, 21, 21, 14, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471244394Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471258494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471271563Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471284933Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Next flush: 43","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471297962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471311012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pidWait for a query_work_queue command to finish before sending IO ":1270,"":"downstairs","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471336591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.4713489Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"0","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.47136225Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"1","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471375519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"client":"2","":"downstairs","session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471389099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471401449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"7c531171-b196-4ecc-ad1f-e9d128ebe4e2 is now active with session: 08549845-9160-4e2b-bca7-e3e0638531f0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471414298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471426498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270,"session_id":"08549845-9160-4e2b-bca7-e3e0638531f0"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:55.471440437Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 361 len 24576 data: 2 2 2 2 2 2 02/30 Write block 62 len 8192 data: 4 4 03/30 Write block 209 len 36864 data: 2 2 2 2 4 4 4 4 4 04/30 Read block 163 len 28672 05/30 Write block 131 len 28672 data: 3 3 3 5 4 4 4 06/30 Read block 450 len 28672 07/30 Flush 08/30 Flush 09/30 Write block 316 len 12288 data: 4 4 4 10/30 Read block 455 len 24576 11/30 Write block 326 len 24576 data: 2 3 3 3 4 5 12/30 Write block 201 len 24576 data: 2 2 2 3 3 3 13/30 Write block 164 len 16384 data: 4 4 3 3 14/30 Read block 91 len 24576 15/30 Read block 377 len 12288 16/30 Read block 129 len 16384 17/30 Read block 550 len 24576 18/30 Write block 466 len 8192 data: 2 2 19/30 Read block 210 len 20480 20/30 Write block 583 len 24576 data: 4 4 4 5 6 6 21/30 Write block 590 len 20480 data: 6 4 3 3 3 22/30 Flush 23/30 Write block 455 len 36864 data: 2 2 2 2 2 2 2 2 2 24/30 Write block 454 len 40960 data: 2 3 3 3 3 3 3 3 3 3 25/30 Read block 210 len 36864 26/30 Read block 108 len 20480 27/30 Read block 198 len 12288 28/30 Read block 123 len 20480 29/30 Read block 52 len 4096 30/30 Write block 385 len 32768 data: 3 2 2 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 9 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:55.495128662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1270} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 22 22 22 43 43 43 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 20 20 20 41 41 41 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 22 22 22 45 45 45 F F F 9 180-199 21 21 21 42 42 42 F F F 10 200-219 22 22 22 45 45 45 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 21 21 21 42 42 42 F F F 14 280-299 17 17 17 36 36 36 F F F 15 300-319 22 22 22 45 45 45 F F F 16 320-339 22 22 22 45 45 45 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 19 19 19 38 38 38 T T T 20 400-419 19 19 19 38 38 38 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 9 9 9 19 19 19 T T T 23 460-479 22 22 22 45 45 45 T T T 24 480-499 19 19 19 38 38 38 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 22 22 22 45 45 45 F F F Max gen: 22, Max flush: 45 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 8 8 8 17 17 17 F F F 3 060-079 22 22 22 43 43 43 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 20 20 20 41 41 41 F F F 6 120-139 22 22 22 43 43 43 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 22 22 22 45 45 45 F F F 9 180-199 21 21 21 42 42 42 F F F 10 200-219 22 22 22 45 45 45 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 21 21 21 42 42 42 F F F 14 280-299 17 17 17 36 36 36 F F F 15 300-319 22 22 22 45 45 45 F F F 16 320-339 22 22 22 45 45 45 F F F 17 340-359 19 19 19 38 38 38 F F F 18 360-379 22 22 22 43 43 43 F F F 19 380-399 23 23 23 46 46 46 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 14 14 14 27 27 27 F F F 22 440-459 23 23 23 46 46 46 F F F 23 460-479 23 23 23 46 46 46 F F F 24 480-499 19 19 19 38 38 38 F F F 25 500-519 21 21 21 42 42 42 F F F 26 520-539 20 20 20 41 41 41 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 22 22 22 45 45 45 F F F Max gen: 23, Max flush: 46 Begin loop 12 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.814164979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.858249968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.858859386Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.858895495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.858910254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.858924984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} {"msg":"Crucible 9fd8b42b-1dac-46a5-8d84-59a90677ef14 has session id: b0e9b440-749b-49a2-957b-cabfb2e4bcd5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.859361028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Upstairs opts: Upstairs UUID: 9fd8b42b-1dac-46a5-8d84-59a90677ef14, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.859402506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Crucible stats registered with UUID: 9fd8b42b-1dac-46a5-8d84-59a90677ef14","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.859427665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889255149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889290437Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} {"msg":"9fd8b42b-1dac-46a5-8d84-59a90677ef14 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889358035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889377054Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889399724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889414063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889438182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889452082Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889466021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889479321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.88949312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.88950676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:57517)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889654024Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"io task","client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:37028)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889702913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"io task","client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_connection connected from Ok(127.0.0.1:40935)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.889790139Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"io task","client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.890384058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.890428906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.890462255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.890491014Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.890582951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.890676237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891124421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891206478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891241747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891891074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891919353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891934172Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891950991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.891965271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.89197875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.8919912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892022399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[0]R flush_numbers[0..12]: [42, 41, 17, 43, 33, 41, 43, 23, 45, 42, 45, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892037398Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[0]R generation[0..12]: [21, 20, 8, 22, 16, 20, 22, 11, 22, 21, 22, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892051288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892065537Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[1]R flush_numbers[0..12]: [42, 41, 17, 43, 33, 41, 43, 23, 45, 42, 45, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892079327Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[1]R generation[0..12]: [21, 20, 8, 22, 16, 20, 22, 11, 22, 21, 22, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892093126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892107226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[2]R flush_numbers[0..12]: [42, 41, 17, 43, 33, 41, 43, 23, 45, 42, 45, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892120805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[2]R generation[0..12]: [21, 20, 8, 22, 16, 20, 22, 11, 22, 21, 22, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892134465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs"Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892156414Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892172193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892185873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Next flush: 47","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892198943Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892212172Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"":"downstairs","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892225811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892237891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"0","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892251301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"1","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.89226475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"client":"2","":"downstairs","session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.89227844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892290749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"9fd8b42b-1dac-46a5-8d84-59a90677ef14 is now active with session: b0e9b440-749b-49a2-957b-cabfb2e4bcd5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892303649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892315598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278,"session_id":"b0e9b440-749b-49a2-957b-cabfb2e4bcd5"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:03:59.892328108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 56 len 20480 data: 3 3 3 3 3 02/30 Flush 03/30 Write block 190 len 4096 data: 3 04/30 Read block 297 len 36864 05/30 Read block 539 len 40960 06/30 Write block 386 len 4096 data: 3 07/30 Read block 165 len 16384 08/30 Write block 133 len 20480 data: 4 6 5 5 5 09/30 Write block 437 len 16384 data: 2 3 3 3 10/30 Flush 11/30 Write block 342 len 32768 data: 5 3 3 2 2 2 2 2 12/30 Write block 106 len 32768 data: 2 2 2 3 3 3 3 4 13/30 Write block 358 len 24576 data: 2 2 2 3 3 3 14/30 Read block 195 len 12288 15/30 Write block 197 len 16384 data: 2 2 2 2 16/30 Write block 162 len 4096 data: 5 17/30 Read block 91 len 28672 18/30 Read block 307 len 8192 19/30 Flush 20/30 Write block 468 len 16384 data: 3 3 4 6 21/30 Read block 157 len 4096 22/30 Read block 309 len 32768 23/30 Write block 461 len 28672 data: 4 4 4 2 2 3 3 24/30 Read block 50 len 24576 25/30 Write block 278 len 40960 data: 2 2 2 2 4 4 3 3 4 4 26/30 Write block 514 len 36864 data: 4 3 3 3 3 3 4 5 5 27/30 Read block 81 len 8192 28/30 Write block 434 len 32768 data: 3 3 3 3 4 4 4 3 29/30 Write block 47 len 8192 data: 4 4 30/30 Write block 494 len 24576 data: 3 2 3 3 3 4 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 8 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:03:59.916124269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1278} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 24 24 24 47 47 47 T T T 3 060-079 24 24 24 47 47 47 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 F F F 6 120-139 24 24 24 48 48 48 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 24 24 24 49 49 49 F F F 9 180-199 24 24 24 49 49 49 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 21 21 21 42 42 42 T T T 14 280-299 17 17 17 36 36 36 T T T 15 300-319 22 22 22 45 45 45 F F F 16 320-339 22 22 22 45 45 45 F F F 17 340-359 24 24 24 49 49 49 F F F 18 360-379 24 24 24 49 49 49 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 24 24 24 48 48 48 T T T 22 440-459 24 24 24 48 48 48 T T T 23 460-479 23 23 23 46 46 46 T T T 24 480-499 19 19 19 38 38 38 T F T <--- 25 500-519 21 21 21 42 42 42 T T T 26 520-539 20 20 20 41 41 41 T T T 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 22 22 22 45 45 45 F F F Max gen: 24, Max flush: 49 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 25 25 25 50 50 50 F F F 3 060-079 24 24 24 47 47 47 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 F F F 6 120-139 24 24 24 48 48 48 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 24 24 24 49 49 49 F F F 9 180-199 24 24 24 49 49 49 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 19 19 19 38 38 38 F F F 12 240-259 16 16 16 35 35 35 F F F 13 260-279 25 25 25 50 50 50 F F F 14 280-299 25 25 25 50 50 50 F F F 15 300-319 22 22 22 45 45 45 F F F 16 320-339 22 22 22 45 45 45 F F F 17 340-359 24 24 24 49 49 49 F F F 18 360-379 24 24 24 49 49 49 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 25 25 25 50 50 50 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 25 25 25 50 50 50 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 25 25 25 50 50 50 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 22 22 22 45 45 45 F F F Max gen: 25, Max flush: 50 Begin loop 13 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.276518563Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:04.318491078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.318900693Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.349778779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350322789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350361817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350377367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350392156Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} {"msg":"Crucible aaa5365f-c618-4660-9d17-5267a5f63662 has session id: af1c6010-5717-49b3-8725-19d6706d2f26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350922787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Upstairs opts: Upstairs UUID: aaa5365f-c618-4660-9d17-5267a5f63662, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350961036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Crucible stats registered with UUID: aaa5365f-c618-4660-9d17-5267a5f63662","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.350978475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.381782123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.381814872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} {"msg":"aaa5365f-c618-4660-9d17-5267a5f63662 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.381893159Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.381917349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.381946377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.381975446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382039034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382070833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382103052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382136381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382173879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382210258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_connection connected from Ok(127.0.0.1:38728)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382332283Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"io task","client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_connection connected from Ok(127.0.0.1:62426)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382377272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"io task","client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_connection connected from Ok(127.0.0.1:45644)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.382408881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"io task","client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383066707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383108555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383142284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383175833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383206962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383230151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383717104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383834679Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.383903747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384277383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384304102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384318942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384470726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384495505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384509855Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384522315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384535714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[0]R flush_numbers[0..12]: [42, 41, 50, 47, 33, 49, 48, 23, 49, 49, 49, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384549683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[0]R generation[0..12]: [21, 20, 25, 24, 16, 24, 24, 11, 24, 24, 24, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384563633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384577782Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[1]R flush_numbers[0..12]: [42, 41, 50, 47, 33, 49, 48, 23, 49, 49, 49, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384591592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[1]R generation[0..12]: [21, 20, 25, 24, 16, 24, 24, 11, 24, 24, 24, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384605192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384619161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[2]R flush_numbers[0..12]: [42, 41, 50, 47, 33, 49, 48, 23, 49, 49, 49, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.38463334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[2]R generation[0..12]: [21, 20, 25, 24, 16, 24, 24, 11, 24, 24, 24, 19]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.38464733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384661479Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Max found gen is 26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384674609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384689868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Next flush: 51","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-07T14:04:04.384703048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384791335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"":"downstairs","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384808084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384820404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"0","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384834133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"1","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384847673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"client":"2","":"downstairs","session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384861212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384873742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"aaa5365f-c618-4660-9d17-5267a5f63662 is now active with session: af1c6010-5717-49b3-8725-19d6706d2f26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384886841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.384898931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286,"session_id":"af1c6010-5717-49b3-8725-19d6706d2f26"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:04.3849114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 582 len 32768 data: 2 5 5 5 6 7 7 6 02/30 Read block 289 len 16384 03/30 Write block 227 len 28672 data: 3 3 4 3 4 3 3 04/30 Flush 05/30 Write block 283 len 36864 data: 5 4 4 5 5 4 3 3 4 06/30 Read block 587 len 16384 07/30 Read block 533 len 8192 08/30 Write block 177 len 24576 data: 2 2 3 3 3 3 09/30 Flush 10/30 Write block 372 len 8192 data: 4 4 11/30 Write block 265 len 24576 data: 2 2 3 3 4 5 12/30 Read block 443 len 24576 13/30 Write block 525 len 40960 data: 5 5 4 4 4 4 3 3 2 2 14/30 Flush 15/30 Write block 255 len 24576 data: 3 3 2 2 2 2 16/30 Write block 317 len 24576 data: 5 5 4 4 3 3 17/30 Write block 238 len 32768 data: 2 4 4 3 3 3 3 3 18/30 Read block 470 len 36864 19/30 Write block 491 len 16384 data: 3 3 3 4 20/30 Write block 191 len 28672 data: 3 3 4 3 2 2 3 21/30 Read block 107 len 28672 22/30 Flush 23/30 Write block 585 len 4096 data: 6 24/30 Flush 25/30 Write block 53 len 8192 data: 4 4 26/30 Flush 27/30 Write block 453 len 12288 data: 2 3 4 28/30 Read block 486 len 4096 29/30 Read block 387 len 32768 30/30 Read block 490 len 8192 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 3 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:04.414272981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1286} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 26 26 26 56 56 56 F F F 3 060-079 24 24 24 47 47 47 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 F F F 6 120-139 24 24 24 48 48 48 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 26 26 26 54 54 54 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 26 26 26 54 54 54 F F F 16 320-339 26 26 26 54 54 54 F F F 17 340-359 24 24 24 49 49 49 F F F 18 360-379 26 26 26 53 53 53 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 25 25 25 50 50 50 T T T 23 460-479 25 25 25 50 50 50 F F F 24 480-499 26 26 26 54 54 54 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 26 26 26 53 53 53 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 26 26 26 55 55 55 F F F Max gen: 26, Max flush: 56 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 26 26 26 56 56 56 F F F 3 060-079 24 24 24 47 47 47 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 F F F 6 120-139 24 24 24 48 48 48 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 26 26 26 54 54 54 F F F 14 280-299 26 26 26 52 52 52 F F F 15 300-319 26 26 26 54 54 54 F F F 16 320-339 26 26 26 54 54 54 F F F 17 340-359 24 24 24 49 49 49 F F F 18 360-379 26 26 26 53 53 53 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 27 27 27 57 57 57 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 26 26 26 54 54 54 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 26 26 26 53 53 53 F F F 27 540-559 16 16 16 35 35 35 F F F 28 560-579 19 19 19 38 38 38 F F F 29 580-599 26 26 26 55 55 55 F F F Max gen: 27, Max flush: 57 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.706637272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:08.749417208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.749803984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.780998037Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.781550837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.781597255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.781616294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.781634804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} {"msg":"Crucible 0b6b8cc1-5233-4fba-ab42-648b90909069 has session id: 581134fc-5fc8-4e17-9c98-c69894b8673c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.782073818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Upstairs opts: Upstairs UUID: 0b6b8cc1-5233-4fba-ab42-648b90909069, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.782123616Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Crucible stats registered with UUID: 0b6b8cc1-5233-4fba-ab42-648b90909069","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.782148655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812230569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812277158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} {"msg":"0b6b8cc1-5233-4fba-ab42-648b90909069 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812311786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812344375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812391443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812429872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812465411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.81250051Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812536858Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812557648Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812589076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812623145Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64414)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812668333Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"io task","client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_connection connected from Ok(127.0.0.1:64679)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812860677Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"io task","client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_connection connected from Ok(127.0.0.1:34701)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.812906095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"io task","client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.813349019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.813383148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.813519723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.813547882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.813563091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.813577331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814077962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814104602Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814366072Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.81468832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814716599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814743018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814767768Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814793917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814811836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814838615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814855994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[0]R flush_numbers[0..12]: [42, 41, 56, 47, 33, 49, 48, 23, 52, 54, 49, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814896093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[0]R generation[0..12]: [21, 20, 26, 24, 16, 24, 24, 11, 26, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814913452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814927642Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[1]R flush_numbers[0..12]: [42, 41, 56, 47, 33, 49, 48, 23, 52, 54, 49, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814941551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[1]R generation[0..12]: [21, 20, 26, 24, 16, 24, 24, 11, 26, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.814955211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.81496913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[2]R flush_numbers[0..12]: [42, 41, 56, 47, 33, 49, 48, 23, 52, 54, 49, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.81498293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[2]R generation[0..12]: [21, 20, 26, 24, 16, 24, 24, 11, 26, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.81499656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815010649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815023939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815037188Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Next flush: 58","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815050307Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815063537Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"":"downstairs","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815076757Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:04:08.815091096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"0","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815108715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"1","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815121795Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"client":"2","":"downstairs","session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815135044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815147384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"0b6b8cc1-5233-4fba-ab42-648b90909069 is now active with session: 581134fc-5fc8-4e17-9c98-c69894b8673c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815160003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815171913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294,"session_id":"581134fc-5fc8-4e17-9c98-c69894b8673c"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:08.815184273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 303 len 24576 data: 2 2 4 4 4 5 02/30 Read block 505 len 4096 03/30 Write block 342 len 32768 data: 6 4 4 3 3 3 3 3 04/30 Read block 404 len 28672 05/30 Write block 282 len 16384 data: 5 6 5 5 06/30 Write block 538 len 24576 data: 2 2 2 2 2 2 07/30 Read block 324 len 24576 08/30 Flush 09/30 Flush 10/30 Flush 11/30 Write block 482 len 28672 data: 3 3 3 3 3 3 4 12/30 Write block 369 len 16384 data: 3 4 4 5 13/30 Read block 276 len 32768 14/30 Write block 264 len 36864 data: 2 3 3 4 4 5 6 5 5 15/30 Write block 58 len 12288 data: 4 4 4 16/30 Flush 17/30 Flush 18/30 Write block 577 len 36864 data: 3 3 3 3 2 3 6 6 7 19/30 Write block 316 len 28672 data: 5 6 6 5 5 4 4 20/30 Flush 21/30 Write block 584 len 40960 data: 7 8 7 8 8 7 7 5 4 4 22/30 Write block 492 len 12288 data: 4 4 5 23/30 Flush 24/30 Write block 374 len 24576 data: 3 3 2 2 2 2 25/30 Read block 28 len 8192 26/30 Read block 507 len 20480 27/30 Write block 544 len 4096 data: 2 28/30 Read block 420 len 12288 29/30 Write block 265 len 32768 data: 4 4 5 5 6 7 6 6 30/30 Read block 357 len 32768 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 6 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 8 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 6 7 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:08.848073485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1294} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 28 28 28 61 61 61 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 F F F 6 120-139 24 24 24 48 48 48 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 28 28 28 61 61 61 T T T 14 280-299 28 28 28 58 58 58 F F F 15 300-319 28 28 28 63 63 63 F F F 16 320-339 28 28 28 63 63 63 F F F 17 340-359 28 28 28 58 58 58 F F F 18 360-379 28 28 28 61 61 61 T T T 19 380-399 24 24 24 48 48 48 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 27 27 27 57 57 57 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 28 28 28 64 64 64 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 28 28 28 58 58 58 T T T 28 560-579 28 28 28 63 63 63 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 28, Max flush: 64 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 28 28 28 61 61 61 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 F F F 6 120-139 24 24 24 48 48 48 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 26 26 26 54 54 54 F F F 10 200-219 24 24 24 49 49 49 F F F 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 29 29 29 65 65 65 F F F 14 280-299 28 28 28 58 58 58 F F F 15 300-319 28 28 28 63 63 63 F F F 16 320-339 28 28 28 63 63 63 F F F 17 340-359 28 28 28 58 58 58 F F F 18 360-379 29 29 29 65 65 65 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 19 19 19 38 38 38 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 27 27 27 57 57 57 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 28 28 28 64 64 64 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 65 65 65 F F F 28 560-579 28 28 28 63 63 63 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 29, Max flush: 65 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.159776439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.203548589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204170826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204210805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204227874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204245903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} {"msg":"Crucible 864dc3f2-3ea2-413f-9308-a58cca89abde has session id: fa5d7d0d-579b-4974-bb70-633c5eaa868a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204677108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Upstairs opts: Upstairs UUID: 864dc3f2-3ea2-413f-9308-a58cca89abde, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204727286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Crucible stats registered with UUID: 864dc3f2-3ea2-413f-9308-a58cca89abde","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.204749335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.23482449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.234870468Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} {"msg":"864dc3f2-3ea2-413f-9308-a58cca89abde active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.234939935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.234969185Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.234997424Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235030042Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235059921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235144088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235185797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235222645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.23536631Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235413809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38844)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235449467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"io task","client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_connection connected from Ok(127.0.0.1:45730)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235472966Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"io task","client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38220)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235497095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"io task","client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235911811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235946749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235965158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.235993117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.236085344Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.236124613Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.236606215Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.236635104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.236793329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237283171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237322769Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237343039Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237356278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237370258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237383687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237396217Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237409826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[0]R flush_numbers[0..12]: [42, 41, 61, 61, 33, 49, 48, 23, 52, 54, 49, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237423626Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[0]R generation[0..12]: [21, 20, 28, 28, 16, 24, 24, 11, 26, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237437635Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237451855Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[1]R flush_numbers[0..12]: [42, 41, 61, 61, 33, 49, 48, 23, 52, 54, 49, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237468144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[1]R generation[0..12]: [21, 20, 28, 28, 16, 24, 24, 11, 26, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237482264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237496453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[2]R flush_numbers[0..12]: [42, 41, 61, 61, 33, 49, 48, 23, 52, 54, 49, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237510123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[2]R generation[0..12]: [21, 20, 28, 28, 16, 24, 24, 11, 26, 26, 24, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237523662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237537392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237550331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237563471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Next flush: 66","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.23757644Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.23758949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"":"downstairs","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:04:13.237602839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237622069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"0","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237635348Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"1","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237648478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"client":"2","":"downstairs","session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237661797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237676347Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"864dc3f2-3ea2-413f-9308-a58cca89abde is now active with session: fa5d7d0d-579b-4974-bb70-633c5eaa868a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237689466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237701556Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302,"session_id":"fa5d7d0d-579b-4974-bb70-633c5eaa868a"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:13.237713745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 32768 data: 6 5 5 5 4 3 3 4 02/30 Write block 370 len 20480 data: 5 5 6 5 4 03/30 Read block 468 len 8192 04/30 Write block 410 len 24576 data: 4 3 3 3 3 2 05/30 Flush 06/30 Write block 560 len 36864 data: 2 2 3 3 3 6 6 7 6 07/30 Write block 181 len 16384 data: 4 4 4 3 08/30 Read block 266 len 4096 09/30 Read block 444 len 24576 10/30 Read block 232 len 28672 11/30 Write block 115 len 24576 data: 3 3 3 3 3 3 12/30 Read block 422 len 8192 13/30 Write block 364 len 4096 data: 3 14/30 Write block 319 len 12288 data: 6 6 5 15/30 Write block 367 len 36864 data: 2 3 4 6 6 7 6 5 4 16/30 Write block 201 len 12288 data: 3 3 3 17/30 Read block 298 len 20480 18/30 Read block 549 len 40960 19/30 Read block 6 len 40960 20/30 Read block 263 len 40960 21/30 Read block 536 len 8192 22/30 Read block 45 len 8192 23/30 Write block 281 len 12288 data: 3 6 7 24/30 Read block 115 len 24576 25/30 Read block 157 len 32768 26/30 Read block 11 len 36864 27/30 Read block 572 len 40960 28/30 Write block 453 len 4096 data: 3 29/30 Read block 568 len 4096 30/30 Write block 295 len 36864 data: 2 2 2 2 2 2 3 2 3 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 9 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 6 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Read 5 Done Done Done false 1017 Acked Read 10 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 24 24 24 72 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1004 1004 1004 Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:13.260235362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1302} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 28 28 28 61 61 61 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 24 24 24 49 49 49 T T T 6 120-139 24 24 24 48 48 48 T T T 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 26 26 26 54 54 54 T T T 10 200-219 24 24 24 49 49 49 T T T 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 29 29 29 65 65 65 F F F 14 280-299 28 28 28 58 58 58 T T T 15 300-319 28 28 28 63 63 63 T T T 16 320-339 28 28 28 63 63 63 T T T 17 340-359 28 28 28 58 58 58 F F F 18 360-379 30 30 30 66 66 66 T T T 19 380-399 24 24 24 48 48 48 F F F 20 400-419 30 30 30 66 66 66 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 27 27 27 57 57 57 T T T 23 460-479 25 25 25 50 50 50 F F F 24 480-499 28 28 28 64 64 64 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 65 65 65 F F F 28 560-579 30 30 30 66 66 66 T T T 29 580-599 28 28 28 64 64 64 F F F Max gen: 30, Max flush: 66 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 F F F 2 040-059 28 28 28 61 61 61 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 31 31 31 67 67 67 F F F 6 120-139 31 31 31 67 67 67 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 31 31 31 67 67 67 F F F 10 200-219 31 31 31 67 67 67 F F F 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 29 29 29 65 65 65 F F F 14 280-299 31 31 31 67 67 67 F F F 15 300-319 31 31 31 67 67 67 F F F 16 320-339 31 31 31 67 67 67 F F F 17 340-359 28 28 28 58 58 58 F F F 18 360-379 31 31 31 67 67 67 F F F 19 380-399 24 24 24 48 48 48 F F F 20 400-419 30 30 30 66 66 66 F F F 21 420-439 25 25 25 50 50 50 F F F 22 440-459 31 31 31 67 67 67 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 28 28 28 64 64 64 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 65 65 65 F F F 28 560-579 31 31 31 67 67 67 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 31, Max flush: 67 Begin loop 16 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.642061124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.686333025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.686904395Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.686938574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.686953663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.686968003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} {"msg":"Crucible 573162e6-2021-42ae-aaa8-7f0b94d0eb41 has session id: 7c4e86a3-063d-4336-8fc1-a73f3efd6d80","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.687464885Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Upstairs opts: Upstairs UUID: 573162e6-2021-42ae-aaa8-7f0b94d0eb41, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.687500003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Crucible stats registered with UUID: 573162e6-2021-42ae-aaa8-7f0b94d0eb41","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.687515593Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.71807335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718105689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} {"msg":"573162e6-2021-42ae-aaa8-7f0b94d0eb41 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718177396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718212435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718242794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718274403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718313531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.71834826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718403248Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718648649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718694707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718771794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_connection connected from Ok(127.0.0.1:49257)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718810943Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"io task","client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_connection connected from Ok(127.0.0.1:56073)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718850982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"io task","client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_connection connected from Ok(127.0.0.1:63367)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.718871551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"io task","client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719092543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719119862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719262547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719298766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719328014Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719358523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719764409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719844746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.719901304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720408466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720433654Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720448634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720461204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720474523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720487752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720499852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720513042Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[0]R flush_numbers[0..12]: [42, 41, 61, 61, 33, 67, 67, 23, 52, 67, 67, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720526681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[0]R generation[0..12]: [21, 20, 28, 28, 16, 31, 31, 11, 26, 31, 31, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720540391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.72055448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[1]R flush_numbers[0..12]: [42, 41, 61, 61, 33, 67, 67, 23, 52, 67, 67, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.72056813Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[1]R generation[0..12]: [21, 20, 28, 28, 16, 31, 31, 11, 26, 31, 31, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720584919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720599388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[2]R flush_numbers[0..12]: [42, 41, 61, 61, 33, 67, 67, 23, 52, 67, 67, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720613038Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[2]R generation[0..12]: [21, 20, 28, 28, 16, 31, 31, 11, 26, 31, 31, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720626667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720640677Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720653897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720667046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Next flush: 68","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720688135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720701175Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"":"downstairs","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720714384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720726594Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"0","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720739774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"1","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720752813Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"client":"2","":"downstairs","session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720766122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720778392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"573162e6-2021-42ae-aaa8-7f0b94d0eb41 is now active with session: 7c4e86a3-063d-4336-8fc1-a73f3efd6d80","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720793321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720805731Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310,"session_id":"7c4e86a3-063d-4336-8fc1-a73f3efd6d80"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:17.720818921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 491 len 20480 data: 4 5 5 6 3 02/30 Write block 316 len 40960 data: 6 7 7 7 7 6 5 3 2 2 03/30 Write block 398 len 32768 data: 5 5 4 3 3 3 2 2 04/30 Read block 67 len 8192 05/30 Read block 388 len 32768 06/30 Read block 1 len 8192 07/30 Write block 280 len 36864 data: 3 4 7 8 6 6 6 6 5 08/30 Write block 266 len 12288 data: 5 6 6 09/30 Read block 202 len 20480 10/30 Write block 432 len 20480 data: 3 3 4 4 4 11/30 Read block 50 len 40960 12/30 Read block 357 len 4096 13/30 Read block 258 len 28672 14/30 Read block 178 len 20480 15/30 Flush 16/30 Write block 487 len 40960 data: 4 5 3 3 5 6 6 7 4 4 17/30 Read block 292 len 24576 18/30 Write block 213 len 4096 data: 5 19/30 Write block 207 len 20480 data: 3 2 3 3 3 20/30 Read block 477 len 12288 21/30 Read block 159 len 40960 22/30 Read block 111 len 4096 23/30 Read block 282 len 16384 24/30 Write block 202 len 16384 data: 4 4 4 4 25/30 Read block 189 len 4096 26/30 Read block 322 len 28672 27/30 Read block 264 len 12288 28/30 Write block 39 len 36864 data: 4 3 3 3 3 2 2 3 5 29/30 Write block 276 len 32768 data: 4 3 3 3 4 5 8 9 30/30 Read block 224 len 16384 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 10 Done Done Done false 1016 Acked Read 6 Done Done Done false 1017 Acked Write 1 Done Done Done false 1018 Acked Write 5 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Write 8 Done Done Done false 1029 Acked Read 4 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 14 14 15 43 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:17.744535595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1310} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 20 20 20 41 41 41 T T T 2 040-059 28 28 28 61 61 61 T T T 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 31 31 31 67 67 67 F F F 6 120-139 31 31 31 67 67 67 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 31 31 31 67 67 67 F F F 10 200-219 31 31 31 67 67 67 T T T 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 32 32 32 68 68 68 T T T 14 280-299 32 32 32 68 68 68 T T T 15 300-319 32 32 32 68 68 68 F F F 16 320-339 32 32 32 68 68 68 F F F 17 340-359 28 28 28 58 58 58 F F F 18 360-379 31 31 31 67 67 67 F F F 19 380-399 32 32 32 68 68 68 F F F 20 400-419 32 32 32 68 68 68 F F F 21 420-439 32 32 32 68 68 68 F F F 22 440-459 31 31 31 67 67 67 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 32 32 32 68 68 68 T T T 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 65 65 65 F F F 28 560-579 31 31 31 67 67 67 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 32, Max flush: 68 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 33 33 33 69 69 69 F F F 2 040-059 33 33 33 69 69 69 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 F F F 5 100-119 31 31 31 67 67 67 F F F 6 120-139 31 31 31 67 67 67 F F F 7 140-159 11 11 11 23 23 23 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 31 31 31 67 67 67 F F F 10 200-219 33 33 33 69 69 69 F F F 11 220-239 26 26 26 54 54 54 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 33 33 33 69 69 69 F F F 14 280-299 33 33 33 69 69 69 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 32 32 32 68 68 68 F F F 17 340-359 28 28 28 58 58 58 F F F 18 360-379 31 31 31 67 67 67 F F F 19 380-399 32 32 32 68 68 68 F F F 20 400-419 32 32 32 68 68 68 F F F 21 420-439 32 32 32 68 68 68 F F F 22 440-459 31 31 31 67 67 67 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 33 33 33 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 F F F 27 540-559 29 29 29 65 65 65 F F F 28 560-579 31 31 31 67 67 67 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 33, Max flush: 69 Begin loop 17 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.070518151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.11347444Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114136486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114184844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114203044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114221213Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} {"msg":"Crucible 365da440-e2f3-408d-9522-8dd4f7b9c727 has session id: 1c134d0b-a5c3-4771-9b71-62ade0a0f5e6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114664957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Upstairs opts: Upstairs UUID: 365da440-e2f3-408d-9522-8dd4f7b9c727, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114741054Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Crucible stats registered with UUID: 365da440-e2f3-408d-9522-8dd4f7b9c727","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.114771663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145415737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145449246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} {"msg":"365da440-e2f3-408d-9522-8dd4f7b9c727 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145501864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145523563Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145551372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145569102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145586581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.14560359Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.14562468Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145641739Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145737925Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145786994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:33858)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.145823052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"io task","client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:37308)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146006536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"io task","client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_connection connected from Ok(127.0.0.1:48987)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146037394Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"io task","client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146457979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146486518Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146609374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146630783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146648783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.146662172Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147156794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147242981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.14726666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147705634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147734903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147749923Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147843869Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147865079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147878818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147891238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147904877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[0]R flush_numbers[0..12]: [42, 69, 69, 61, 33, 67, 67, 23, 52, 67, 69, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147918787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[0]R generation[0..12]: [21, 33, 33, 28, 16, 31, 31, 11, 26, 31, 33, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147932636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147947076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[1]R flush_numbers[0..12]: [42, 69, 69, 61, 33, 67, 67, 23, 52, 67, 69, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147961095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[1]R generation[0..12]: [21, 33, 33, 28, 16, 31, 31, 11, 26, 31, 33, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147974975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.147989084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[2]R flush_numbers[0..12]: [42, 69, 69, 61, 33, 67, 67, 23, 52, 67, 69, 54]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148006043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[2]R generation[0..12]: [21, 33, 33, 28, 16, 31, 31, 11, 26, 31, 33, 26]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148020363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148034413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Max found gen is 34","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148047392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148060901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148073901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:04:22.148086971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"":"downstairs","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.14810548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.14811765Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"0","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148131069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"1","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148144789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"client":"2","":"downstairs","session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148158338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148170957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"365da440-e2f3-408d-9522-8dd4f7b9c727 is now active with session: 1c134d0b-a5c3-4771-9b71-62ade0a0f5e6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148183937Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148196236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318,"session_id":"1c134d0b-a5c3-4771-9b71-62ade0a0f5e6"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:22.148208506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 32768 data: 4 4 2 2 2 2 3 2 02/30 Read block 538 len 8192 03/30 Write block 230 len 4096 data: 4 04/30 Read block 217 len 12288 05/30 Write block 416 len 24576 data: 2 2 3 2 3 3 06/30 Read block 509 len 20480 07/30 Write block 425 len 12288 data: 4 3 2 08/30 Read block 246 len 40960 09/30 Read block 366 len 28672 10/30 Write block 544 len 40960 data: 3 2 4 4 4 4 4 4 4 4 11/30 Write block 426 len 16384 data: 4 3 2 2 12/30 Write block 531 len 40960 data: 4 4 3 3 2 2 2 3 3 3 13/30 Read block 338 len 36864 14/30 Write block 145 len 16384 data: 4 4 4 3 15/30 Read block 116 len 32768 16/30 Read block 231 len 32768 17/30 Write block 80 len 12288 data: 3 3 3 18/30 Read block 493 len 4096 19/30 Write block 441 len 40960 data: 4 2 2 2 2 2 2 2 2 2 20/30 Write block 113 len 28672 data: 5 4 4 4 4 4 4 21/30 Read block 105 len 28672 22/30 Read block 523 len 4096 23/30 Read block 502 len 28672 24/30 Read block 507 len 12288 25/30 Write block 115 len 28672 data: 5 5 5 5 5 4 3 26/30 Read block 135 len 32768 27/30 Write block 37 len 36864 data: 4 4 5 4 4 4 4 3 3 28/30 Write block 356 len 40960 data: 2 2 3 3 3 4 4 4 4 3 29/30 Write block 184 len 16384 data: 4 4 4 4 30/30 Write block 452 len 16384 data: 2 4 4 5 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Read 3 Done Done Done false 1004 Acked Write 6 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 3 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Write 10 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Write 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 3 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 10 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 27 27 27 81 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:22.168009061Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1318} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 33 33 33 69 69 69 T T T 2 040-059 33 33 33 69 69 69 T T T 3 060-079 28 28 28 61 61 61 F F F 4 080-099 16 16 16 33 33 33 T T T 5 100-119 31 31 31 67 67 67 T T T 6 120-139 31 31 31 67 67 67 T T T 7 140-159 11 11 11 23 23 23 T T T 8 160-179 26 26 26 52 52 52 F F F 9 180-199 31 31 31 67 67 67 T T T 10 200-219 33 33 33 69 69 69 F F F 11 220-239 26 26 26 54 54 54 T T T 12 240-259 26 26 26 54 54 54 F F F 13 260-279 33 33 33 69 69 69 F F F 14 280-299 33 33 33 69 69 69 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 32 32 32 68 68 68 F F F 17 340-359 28 28 28 58 58 58 T T T 18 360-379 31 31 31 67 67 67 T T T 19 380-399 32 32 32 68 68 68 F F F 20 400-419 32 32 32 68 68 68 T T T 21 420-439 32 32 32 68 68 68 T T T 22 440-459 31 31 31 67 67 67 T T T 23 460-479 25 25 25 50 50 50 F F F 24 480-499 33 33 33 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 28 28 28 58 58 58 T T T 27 540-559 29 29 29 65 65 65 T T T 28 560-579 31 31 31 67 67 67 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 33, Max flush: 69 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 34 34 34 70 70 70 F F F 2 040-059 34 34 34 70 70 70 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 34 34 34 70 70 70 F F F 5 100-119 34 34 34 70 70 70 F F F 6 120-139 34 34 34 70 70 70 F F F 7 140-159 34 34 34 70 70 70 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 33 33 33 69 69 69 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 26 26 26 54 54 54 F F F 13 260-279 33 33 33 69 69 69 F F F 14 280-299 33 33 33 69 69 69 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 32 32 32 68 68 68 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 34 34 34 70 70 70 F F F 19 380-399 32 32 32 68 68 68 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 34 34 34 70 70 70 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 33 33 33 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 34 34 34 70 70 70 F F F 27 540-559 34 34 34 70 70 70 F F F 28 560-579 31 31 31 67 67 67 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 34, Max flush: 70 Begin loop 18 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.612380515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:26.654190256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.654570512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.685217336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.685765596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.685802115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.685816405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.685832064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} {"msg":"Crucible a843ab7e-3a58-4951-a5bc-2f1c9f36af5f has session id: 83b06bd9-e80d-4ad8-9778-5f3c27852bb3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.686300097Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Upstairs opts: Upstairs UUID: a843ab7e-3a58-4951-a5bc-2f1c9f36af5f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.686328376Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Crucible stats registered with UUID: a843ab7e-3a58-4951-a5bc-2f1c9f36af5f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.686342826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716466338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716496847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} {"msg":"a843ab7e-3a58-4951-a5bc-2f1c9f36af5f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716570514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716594034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716621073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716643392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716661241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71667794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71669494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716717169Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716750408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716781407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:37034)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.716810906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"io task","client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56584)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71697693Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"io task","client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34002)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717046527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"io task","client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717548199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717581758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717599267Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717636746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717778841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.717821519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718278943Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718309432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718392898Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71889483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718923309Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718938129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718950838Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718964638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718978197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.718990677Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719021396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[0]R flush_numbers[0..12]: [42, 70, 70, 61, 70, 70, 70, 70, 52, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719036255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[0]R generation[0..12]: [21, 34, 34, 28, 34, 34, 34, 34, 26, 34, 33, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719050285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719064524Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[1]R flush_numbers[0..12]: [42, 70, 70, 61, 70, 70, 70, 70, 52, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719078254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[1]R generation[0..12]: [21, 34, 34, 28, 34, 34, 34, 34, 26, 34, 33, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719105023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719120862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[2]R flush_numbers[0..12]: [42, 70, 70, 61, 70, 70, 70, 70, 52, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719142751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[2]R generation[0..12]: [21, 34, 34, 28, 34, 34, 34, 34, 26, 34, 33, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719156581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71917073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Max found gen is 35","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71918408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Generation requested: 36 >= found:35","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.71919748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Next flush: 71","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719210449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719223659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"":"downstairs","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719237008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719248988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"0","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719262167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"1","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719275407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"client":"2","":"downstairs","session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719288726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719301056Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"a843ab7e-3a58-4951-a5bc-2f1c9f36af5f is now active with session: 83b06bd9-e80d-4ad8-9778-5f3c27852bb3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719315905Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719328425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326,"session_id":"83b06bd9-e80d-4ad8-9778-5f3c27852bb3"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:26.719340664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 388 len 32768 data: 4 4 4 4 4 4 5 5 02/30 Write block 251 len 8192 data: 4 4 03/30 Read block 583 len 4096 04/30 Write block 85 len 16384 data: 3 5 5 5 05/30 Write block 544 len 40960 data: 4 3 5 5 5 5 5 5 5 5 06/30 Flush 07/30 Write block 555 len 28672 data: 2 2 2 2 2 3 3 08/30 Write block 254 len 12288 data: 3 4 4 09/30 Read block 361 len 4096 10/30 Read block 563 len 12288 11/30 Write block 330 len 4096 data: 5 12/30 Write block 36 len 40960 data: 4 5 5 6 5 5 5 5 4 4 13/30 Read block 90 len 4096 14/30 Read block 295 len 12288 15/30 Write block 392 len 4096 data: 5 16/30 Read block 518 len 12288 17/30 Read block 146 len 4096 18/30 Read block 439 len 20480 19/30 Write block 534 len 36864 data: 4 3 3 3 4 4 4 3 3 20/30 Read block 143 len 28672 21/30 Write block 552 len 8192 data: 6 6 22/30 Flush 23/30 Write block 90 len 4096 data: 5 24/30 Flush 25/30 Write block 546 len 16384 data: 6 6 6 6 26/30 Read block 465 len 28672 27/30 Write block 333 len 28672 data: 6 6 6 7 6 6 4 28/30 Read block 371 len 28672 29/30 Read block 173 len 24576 30/30 Read block 198 len 4096 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 7 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 6 6 6 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:26.742189239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1326} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 36 36 36 72 72 72 F F F 2 040-059 36 36 36 72 72 72 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 34 34 34 70 70 70 F F F 6 120-139 34 34 34 70 70 70 F F F 7 140-159 34 34 34 70 70 70 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 33 33 33 69 69 69 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 36 36 36 72 72 72 F F F 13 260-279 33 33 33 69 69 69 F F F 14 280-299 33 33 33 69 69 69 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 36 36 36 72 72 72 T T T 17 340-359 34 34 34 70 70 70 F F F 18 360-379 34 34 34 70 70 70 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 34 34 34 70 70 70 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 33 33 33 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 36 36 36 72 72 72 F F F 27 540-559 36 36 36 72 72 72 T T T 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 36, Max flush: 73 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 36 36 36 72 72 72 F F F 2 040-059 36 36 36 72 72 72 F F F 3 060-079 28 28 28 61 61 61 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 34 34 34 70 70 70 F F F 6 120-139 34 34 34 70 70 70 F F F 7 140-159 34 34 34 70 70 70 F F F 8 160-179 26 26 26 52 52 52 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 33 33 33 69 69 69 F F F 11 220-239 34 34 34 70 70 70 F F F 12 240-259 36 36 36 72 72 72 F F F 13 260-279 33 33 33 69 69 69 F F F 14 280-299 33 33 33 69 69 69 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 37 37 37 74 74 74 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 34 34 34 70 70 70 F F F 19 380-399 36 36 36 72 72 72 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 34 34 34 70 70 70 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 33 33 33 69 69 69 F F F 25 500-519 25 25 25 50 50 50 F F F 26 520-539 36 36 36 72 72 72 F F F 27 540-559 37 37 37 74 74 74 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 37, Max flush: 74 Begin loop 19 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.045050712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.088489444Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.089114541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.08915953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.089181939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.089200418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} {"msg":"Crucible 82eb6f43-f0c9-4fd1-b6d8-f47d37ca11f0 has session id: 69424acc-33ee-4749-9c05-b74308897086","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.089599524Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Upstairs opts: Upstairs UUID: 82eb6f43-f0c9-4fd1-b6d8-f47d37ca11f0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.089646902Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Crucible stats registered with UUID: 82eb6f43-f0c9-4fd1-b6d8-f47d37ca11f0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.089671481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119538113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119569872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} {"msg":"82eb6f43-f0c9-4fd1-b6d8-f47d37ca11f0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119585942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119599511Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.11962044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.11963479Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119649269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119662859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119676198Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119699977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119736186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119765175Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_connection connected from Ok(127.0.0.1:53477)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119806964Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"io task","client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_connection connected from Ok(127.0.0.1:52637)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119837163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"io task","client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_connection connected from Ok(127.0.0.1:39267)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.119864851Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"io task","client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.120714401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.12074303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.120758399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.120772269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.120839656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.120860106Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.121310939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.121468674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.12158344Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.121858599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.121886649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.121902588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.121984475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122052363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122077092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122192238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122211567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[0]R flush_numbers[0..12]: [42, 72, 72, 61, 73, 70, 70, 70, 52, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122225956Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[0]R generation[0..12]: [21, 36, 36, 28, 36, 34, 34, 34, 26, 34, 33, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122240036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122254085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[1]R flush_numbers[0..12]: [42, 72, 72, 61, 73, 70, 70, 70, 52, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122267875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[1]R generation[0..12]: [21, 36, 36, 28, 36, 34, 34, 34, 26, 34, 33, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122281624Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122295594Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[2]R flush_numbers[0..12]: [42, 72, 72, 61, 73, 70, 70, 70, 52, 70, 69, 70]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122309183Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[2]R generation[0..12]: [21, 36, 36, 28, 36, 34, 34, 34, 26, 34, 33, 34]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122322683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122340052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122353452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Generation requested: 38 >= found:38","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122366641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Next flush: 75","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122379751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.12239293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"":"downstairs","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.1224064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:04:31.122418609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"0","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122484667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"1","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122517776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"client":"2","":"downstairs","session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122545355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122571854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"82eb6f43-f0c9-4fd1-b6d8-f47d37ca11f0 is now active with session: 69424acc-33ee-4749-9c05-b74308897086","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122592703Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122605553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334,"session_id":"69424acc-33ee-4749-9c05-b74308897086"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:31.122621682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 32768 data: 5 5 5 5 5 5 6 5 02/30 Read block 136 len 20480 03/30 Read block 23 len 40960 04/30 Write block 236 len 20480 data: 2 2 3 5 5 05/30 Write block 331 len 36864 data: 6 6 7 7 7 8 7 7 5 06/30 Write block 165 len 28672 data: 5 4 4 4 5 5 4 07/30 Write block 139 len 24576 data: 2 2 2 2 3 3 08/30 Read block 189 len 8192 09/30 Read block 547 len 20480 10/30 Write block 456 len 16384 data: 4 4 4 4 11/30 Write block 360 len 20480 data: 4 5 5 5 5 12/30 Read block 507 len 24576 13/30 Write block 510 len 12288 data: 5 5 4 14/30 Read block 122 len 16384 15/30 Read block 161 len 4096 16/30 Read block 460 len 4096 17/30 Write block 203 len 16384 data: 5 5 5 4 18/30 Write block 57 len 40960 data: 4 5 5 5 3 5 5 3 3 3 19/30 Write block 277 len 28672 data: 4 4 4 5 6 9 10 20/30 Write block 133 len 16384 data: 5 7 6 6 21/30 Write block 363 len 28672 data: 6 6 4 3 3 4 5 22/30 Write block 381 len 24576 data: 3 3 3 3 4 4 23/30 Read block 436 len 40960 24/30 Read block 65 len 28672 25/30 Flush 26/30 Write block 494 len 28672 data: 8 5 5 4 4 6 6 27/30 Read block 174 len 40960 28/30 Read block 16 len 32768 29/30 Read block 374 len 12288 30/30 Read block 137 len 28672 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 10 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 5 4 4 13 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:31.145347892Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1334} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 36 36 36 72 72 72 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 34 34 34 70 70 70 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 38 38 38 75 75 75 F F F 11 220-239 38 38 38 75 75 75 F F F 12 240-259 38 38 38 75 75 75 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 38 38 38 75 75 75 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 38 38 38 75 75 75 F F F 19 380-399 38 38 38 75 75 75 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 38 38 38 75 75 75 T T T 25 500-519 38 38 38 75 75 75 T T T 26 520-539 36 36 36 72 72 72 F F F 27 540-559 37 37 37 74 74 74 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 38, Max flush: 75 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 F F F 1 020-039 36 36 36 72 72 72 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 36 36 36 73 73 73 F F F 5 100-119 34 34 34 70 70 70 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 38 38 38 75 75 75 F F F 11 220-239 38 38 38 75 75 75 F F F 12 240-259 38 38 38 75 75 75 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 38 38 38 75 75 75 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 38 38 38 75 75 75 F F F 19 380-399 38 38 38 75 75 75 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 39 39 39 76 76 76 F F F 25 500-519 39 39 39 76 76 76 F F F 26 520-539 36 36 36 72 72 72 F F F 27 540-559 37 37 37 74 74 74 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 39, Max flush: 76 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.420635848Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.464019232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.464655589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.464701627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.464720167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.464738346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} {"msg":"Crucible f7697ad0-bdca-496d-b0fc-33c583338ed6 has session id: 9968a65b-4689-4068-9960-e991bf5d8086","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.465208179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Upstairs opts: Upstairs UUID: f7697ad0-bdca-496d-b0fc-33c583338ed6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.465258567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Crucible stats registered with UUID: f7697ad0-bdca-496d-b0fc-33c583338ed6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.465293776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495043222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495082341Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} {"msg":"f7697ad0-bdca-496d-b0fc-33c583338ed6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.49511312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495144128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495190827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495228425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495263984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495297443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495333052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495361171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.4953796Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495397019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_connection connected from Ok(127.0.0.1:51889)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495429568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"io task","client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_connection connected from Ok(127.0.0.1:52375)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495464147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"io task","client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_connection connected from Ok(127.0.0.1:63127)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.495501055Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"io task","client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496110824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496150222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496181081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.49621204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496365464Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496402463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496791419Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.496820598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.49705299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497481194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497521853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497553841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497579851Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.49760344Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497631549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497654548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497680397Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[0]R flush_numbers[0..12]: [42, 72, 75, 75, 73, 70, 75, 75, 75, 70, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497705206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[0]R generation[0..12]: [21, 36, 38, 38, 36, 34, 38, 38, 38, 34, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497729145Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497755144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[1]R flush_numbers[0..12]: [42, 72, 75, 75, 73, 70, 75, 75, 75, 70, 75, 75]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497782883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[1]R generation[0..12]: [21, 36, 38, 38, 36, 34, 38, 38, 38, 34, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497809272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497835171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[2]R flush_numbers[0..12]: [42, 72, 75, 75, 73, 70, 75, 75, 75, 70, 75, 75]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:04:35.49786399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[2]R generation[0..12]: [21, 36, 38, 38, 36, 34, 38, 38, 38, 34, 38, 38]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497894159Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497925428Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497953747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.497982136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Next flush: 77","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498010655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498043454Z","hostname"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"":"downstairs","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498077973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498105061Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"0","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498121391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"1","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498134931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"client":"2","":"downstairs","session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.49814846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.49816111Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"f7697ad0-bdca-496d-b0fc-33c583338ed6 is now active with session: 9968a65b-4689-4068-9960-e991bf5d8086","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498174059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498186318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342,"session_id":"9968a65b-4689-4068-9960-e991bf5d8086"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:35.498198658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 375 len 20480 data: 5 3 3 3 3 02/30 Flush 03/30 Write block 458 len 8192 data: 5 5 04/30 Write block 363 len 8192 data: 7 7 05/30 Write block 540 len 16384 data: 5 4 4 3 06/30 Read block 0 len 4096 07/30 Read block 484 len 36864 08/30 Read block 87 len 28672 09/30 Read block 594 len 16384 10/30 Write block 244 len 40960 data: 4 4 3 3 4 3 3 5 5 4 11/30 Write block 13 len 8192 data: 3 3 12/30 Write block 447 len 4096 data: 3 13/30 Write block 209 len 16384 data: 4 4 4 3 14/30 Write block 37 len 8192 data: 6 6 15/30 Read block 542 len 4096 16/30 Read block 136 len 32768 17/30 Write block 322 len 16384 data: 6 4 3 3 18/30 Read block 424 len 36864 19/30 Write block 504 len 32768 data: 6 7 6 4 4 4 6 6 20/30 Write block 369 len 8192 data: 6 7 21/30 Write block 88 len 40960 data: 6 5 6 4 4 5 5 5 2 2 22/30 Read block 268 len 16384 23/30 Read block 278 len 40960 24/30 Write block 222 len 4096 data: 4 25/30 Read block 283 len 4096 26/30 Write block 216 len 16384 data: 5 5 5 5 27/30 Read block 595 len 12288 28/30 Read block 549 len 8192 29/30 Write block 329 len 8192 data: 4 6 30/30 Write block 206 len 8192 data: 5 4 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 2 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Write 4 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 10 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 8 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:35.516096802Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1342} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 42 42 42 T T T 1 020-039 36 36 36 72 72 72 T T T 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 36 36 36 73 73 73 T T T 5 100-119 34 34 34 70 70 70 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 38 38 38 75 75 75 T T T 11 220-239 38 38 38 75 75 75 T T T 12 240-259 38 38 38 75 75 75 T T T 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 38 38 38 75 75 75 T T T 17 340-359 34 34 34 70 70 70 F F F 18 360-379 40 40 40 77 77 77 T T T 19 380-399 38 38 38 75 75 75 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 38 38 38 75 75 75 T T T 23 460-479 25 25 25 50 50 50 F F F 24 480-499 39 39 39 76 76 76 F F F 25 500-519 39 39 39 76 76 76 T T T 26 520-539 36 36 36 72 72 72 F F F 27 540-559 37 37 37 74 74 74 T T T 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 40, Max flush: 77 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 41 41 41 78 78 78 F F F 1 020-039 41 41 41 78 78 78 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 41 41 41 78 78 78 F F F 5 100-119 34 34 34 70 70 70 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 F F F 9 180-199 34 34 34 70 70 70 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 41 41 41 78 78 78 F F F 12 240-259 41 41 41 78 78 78 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 41 41 41 78 78 78 F F F 19 380-399 38 38 38 75 75 75 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 41 41 41 78 78 78 F F F 23 460-479 25 25 25 50 50 50 F F F 24 480-499 39 39 39 76 76 76 F F F 25 500-519 41 41 41 78 78 78 F F F 26 520-539 36 36 36 72 72 72 F F F 27 540-559 41 41 41 78 78 78 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 F F F Max gen: 41, Max flush: 78 Begin loop 21 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.905400532Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.950396048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.950963787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.951007346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.951029575Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.951045225Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} {"msg":"Crucible 617aba08-d43e-4ce6-85d4-7fbf8ef0082a has session id: c9a0c772-27db-4fde-959d-b48288cdded7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.951527817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Upstairs opts: Upstairs UUID: 617aba08-d43e-4ce6-85d4-7fbf8ef0082a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.951570586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Crucible stats registered with UUID: 617aba08-d43e-4ce6-85d4-7fbf8ef0082a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.951599654Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.981779865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.981822823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} {"msg":"617aba08-d43e-4ce6-85d4-7fbf8ef0082a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.981839923Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.981855932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.981886441Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.98192028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.981969578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982006897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982042706Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982076204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982113823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982149022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_connection connected from Ok(127.0.0.1:61493)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982180921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"io task","client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_connection connected from Ok(127.0.0.1:57156)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982214229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"io task","client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50621)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982351364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"io task","client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982837377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982866206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982882085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982896345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982909804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.982926074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.983410236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.983448105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.983561671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.983912068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.983932817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.983946907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984031934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984050713Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984067782Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.98413511Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984153839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[0]R flush_numbers[0..12]: [78, 78, 75, 75, 78, 70, 75, 75, 75, 70, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984167919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[0]R generation[0..12]: [41, 41, 38, 38, 41, 34, 38, 38, 38, 34, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984181658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984195818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[1]R flush_numbers[0..12]: [78, 78, 75, 75, 78, 70, 75, 75, 75, 70, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984209527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[1]R generation[0..12]: [41, 41, 38, 38, 41, 34, 38, 38, 38, 34, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984223087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984236996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[2]R flush_numbers[0..12]: [78, 78, 75, 75, 78, 70, 75, 75, 75, 70, 78, 78]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984250796Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[2]R generation[0..12]: [41, 41, 38, 38, 41, 34, 38, 38, 38, 34, 41, 41]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984264565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984278715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984291894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984305444Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Next flush: 79","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:04:39.984318643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984340463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"":"downstairs","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984353822Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984365792Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"0","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984379191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"1","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984392421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"client":"2","":"downstairs","session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.9844058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.98441823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"617aba08-d43e-4ce6-85d4-7fbf8ef0082a is now active with session: c9a0c772-27db-4fde-959d-b48288cdded7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984431239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984443349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350,"session_id":"c9a0c772-27db-4fde-959d-b48288cdded7"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:39.984455808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 17 len 36864 data: 4 4 5 5 5 6 5 4 5 02/30 Read block 437 len 28672 03/30 Read block 427 len 20480 04/30 Flush 05/30 Write block 592 len 32768 data: 5 5 4 2 2 2 2 2 06/30 Read block 248 len 36864 07/30 Write block 545 len 36864 data: 4 7 7 7 7 6 6 7 7 08/30 Write block 473 len 20480 data: 5 5 5 3 2 09/30 Read block 440 len 8192 10/30 Read block 253 len 16384 11/30 Write block 494 len 16384 data: 9 6 6 5 12/30 Read block 192 len 36864 13/30 Read block 17 len 40960 14/30 Write block 192 len 12288 data: 4 5 4 15/30 Write block 517 len 8192 data: 4 4 16/30 Write block 97 len 28672 data: 3 2 2 2 2 3 3 17/30 Read block 133 len 36864 18/30 Write block 175 len 36864 data: 3 3 3 3 4 4 5 5 5 19/30 Write block 541 len 4096 data: 5 20/30 Write block 531 len 12288 data: 5 5 4 21/30 Read block 491 len 12288 22/30 Write block 230 len 20480 data: 5 5 4 4 2 23/30 Write block 484 len 28672 data: 4 4 4 5 6 4 4 24/30 Write block 480 len 32768 data: 3 3 4 4 5 5 5 6 25/30 Read block 595 len 4096 26/30 Read block 172 len 8192 27/30 Write block 198 len 8192 data: 3 3 28/30 Write block 453 len 36864 data: 5 5 6 5 5 6 6 4 5 29/30 Write block 517 len 20480 data: 5 5 4 5 6 30/30 Write block 222 len 8192 data: 5 3 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 8 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Write 5 Done Done Done false 1008 Acked Read 2 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 4 Done Done Done false 1011 Acked Read 9 Done Done Done false 1012 Acked Read 10 Done Done Done false 1013 Acked Write 3 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 9 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Write 1 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 3 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 23 23 23 69 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1003 1003 1003 Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:40.004766235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1350} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 41 41 41 78 78 78 T T T 5 100-119 34 34 34 70 70 70 T T T 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 38 38 38 75 75 75 T T T 9 180-199 34 34 34 70 70 70 T T T 10 200-219 41 41 41 78 78 78 F F F 11 220-239 41 41 41 78 78 78 T T T 12 240-259 41 41 41 78 78 78 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 41 41 41 78 78 78 F F F 19 380-399 38 38 38 75 75 75 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 41 41 41 78 78 78 T T T 23 460-479 25 25 25 50 50 50 T T T 24 480-499 39 39 39 76 76 76 T T T 25 500-519 41 41 41 78 78 78 T T T 26 520-539 36 36 36 72 72 72 T T T 27 540-559 41 41 41 78 78 78 T T T 28 560-579 36 36 36 72 72 72 F F F 29 580-599 28 28 28 64 64 64 T T T Max gen: 42, Max flush: 79 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 F F F 1 020-039 42 42 42 79 79 79 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 75 75 75 F F F 4 080-099 43 43 43 80 80 80 F F F 5 100-119 43 43 43 80 80 80 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 43 43 43 80 80 80 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 41 41 41 78 78 78 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 41 41 41 78 78 78 F F F 19 380-399 38 38 38 75 75 75 F F F 20 400-419 34 34 34 70 70 70 F F F 21 420-439 34 34 34 70 70 70 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 43 43 43 80 80 80 F F F 24 480-499 43 43 43 80 80 80 F F F 25 500-519 43 43 43 80 80 80 F F F 26 520-539 43 43 43 80 80 80 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 43 43 43 80 80 80 F F F Max gen: 43, Max flush: 80 Begin loop 22 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.4114433Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.454696119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455389024Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455425252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455439852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455454531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} {"msg":"Crucible 57eec13a-38d2-4887-b186-6902d0019e1b has session id: adae987d-0197-4a13-8cb4-213da64ba63c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455918434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Upstairs opts: Upstairs UUID: 57eec13a-38d2-4887-b186-6902d0019e1b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455956883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Crucible stats registered with UUID: 57eec13a-38d2-4887-b186-6902d0019e1b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.455979052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485643021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.4856759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} {"msg":"57eec13a-38d2-4887-b186-6902d0019e1b active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485735438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485765027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485789916Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485804966Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485819265Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485832955Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485846834Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485860334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485874303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485888103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_connection connected from Ok(127.0.0.1:52362)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.485932451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"io task","client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_connection connected from Ok(127.0.0.1:57527)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.4859595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"io task","client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59536)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.486011298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"io task","client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.48680963Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.486851188Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.486882407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.486914056Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.486943365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.486971404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.487499925Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.487530764Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.487547263Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488084284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488109912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488123942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488136751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488150501Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488167521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.48818025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.48819422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[0]R flush_numbers[0..12]: [79, 79, 75, 75, 80, 80, 75, 75, 80, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488208089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[0]R generation[0..12]: [42, 42, 38, 38, 43, 43, 38, 38, 43, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488222288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488236628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[1]R flush_numbers[0..12]: [79, 79, 75, 75, 80, 80, 75, 75, 80, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488250767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[1]R generation[0..12]: [42, 42, 38, 38, 43, 43, 38, 38, 43, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488264887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488279296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[2]R flush_numbers[0..12]: [79, 79, 75, 75, 80, 80, 75, 75, 80, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488293436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[2]R generation[0..12]: [42, 42, 38, 38, 43, 43, 38, 38, 43, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488308135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488322155Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488344324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488357683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Next flush: 81","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488372833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488386293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"":"downstairs","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488399682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488411912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"0","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488425301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"1","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488438561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"client":"2","":"downstairs","session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.48845225Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.48846486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"57eec13a-38d2-4887-b186-6902d0019e1b is now active with session: adae987d-0197-4a13-8cb4-213da64ba63c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488478039Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:04:44.488490419Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358,"session_id":"adae987d-0197-4a13-8cb4-213da64ba63c"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:44.488505308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 24576 data: 4 4 4 4 4 4 02/30 Write block 593 len 8192 data: 6 5 03/30 Read block 285 len 32768 04/30 Write block 403 len 28672 data: 4 3 3 2 2 3 4 05/30 Flush 06/30 Write block 160 len 12288 data: 4 3 6 07/30 Write block 461 len 40960 data: 6 5 5 3 3 4 4 4 4 5 08/30 Read block 520 len 4096 09/30 Read block 425 len 4096 10/30 Write block 30 len 24576 data: 3 4 4 4 5 4 11/30 Read block 207 len 8192 12/30 Write block 586 len 16384 data: 8 9 9 8 13/30 Write block 406 len 16384 data: 3 3 4 5 14/30 Flush 15/30 Write block 280 len 8192 data: 6 7 16/30 Write block 96 len 8192 data: 3 4 17/30 Write block 283 len 40960 data: 11 7 7 7 7 6 4 4 5 4 18/30 Read block 490 len 8192 19/30 Write block 253 len 16384 data: 5 4 5 5 20/30 Read block 370 len 4096 21/30 Write block 58 len 36864 data: 6 6 6 4 6 6 4 4 4 22/30 Read block 344 len 8192 23/30 Read block 285 len 32768 24/30 Read block 220 len 36864 25/30 Read block 503 len 40960 26/30 Write block 423 len 4096 data: 5 27/30 Write block 281 len 4096 data: 8 28/30 Read block 280 len 8192 29/30 Read block 464 len 8192 30/30 Read block 434 len 36864 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 2 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 9 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:44.510857931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1358} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 F F F 1 020-039 44 44 44 82 82 82 F F F 2 040-059 38 38 38 75 75 75 T T T 3 060-079 38 38 38 75 75 75 T T T 4 080-099 43 43 43 80 80 80 T T T 5 100-119 43 43 43 80 80 80 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 41 41 41 78 78 78 T T T 13 260-279 38 38 38 75 75 75 F F F 14 280-299 38 38 38 75 75 75 T T T 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 44 44 44 81 81 81 F F F 19 380-399 44 44 44 81 81 81 F F F 20 400-419 44 44 44 82 82 82 F F F 21 420-439 34 34 34 70 70 70 T T T 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 43 43 43 80 80 80 F F F 25 500-519 43 43 43 80 80 80 F F F 26 520-539 43 43 43 80 80 80 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 44, Max flush: 82 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 F F F 1 020-039 44 44 44 82 82 82 F F F 2 040-059 45 45 45 83 83 83 F F F 3 060-079 45 45 45 83 83 83 F F F 4 080-099 45 45 45 83 83 83 F F F 5 100-119 43 43 43 80 80 80 F F F 6 120-139 38 38 38 75 75 75 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 45 45 45 83 83 83 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 45 45 45 83 83 83 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 44 44 44 81 81 81 F F F 19 380-399 44 44 44 81 81 81 F F F 20 400-419 44 44 44 82 82 82 F F F 21 420-439 45 45 45 83 83 83 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 43 43 43 80 80 80 F F F 25 500-519 43 43 43 80 80 80 F F F 26 520-539 43 43 43 80 80 80 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 45, Max flush: 83 Begin loop 23 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.83346235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.877083765Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.877698953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.877734862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.877749291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.877763941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} {"msg":"Crucible e586d227-55ba-4235-98de-0becb368bcda has session id: d1eb27fd-62a3-4839-b061-048a58cd7369","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.878287212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Upstairs opts: Upstairs UUID: e586d227-55ba-4235-98de-0becb368bcda, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.87832647Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Crucible stats registered with UUID: e586d227-55ba-4235-98de-0becb368bcda","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.87834335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908374476Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908410284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} {"msg":"e586d227-55ba-4235-98de-0becb368bcda active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908430324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908446853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908473802Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908491052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908508141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.90852475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.90854189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908558829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908590948Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908626647Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_connection connected from Ok(127.0.0.1:64286)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908682405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"io task","client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_connection connected from Ok(127.0.0.1:35551)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908722873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"io task","client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_connection connected from Ok(127.0.0.1:55227)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.908952065Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"io task","client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.909535704Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.909575973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.909593962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.909612371Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.90963941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.909666439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.91019384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910333115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910358784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.9107557Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910787079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910802408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910831947Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910846807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910860146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910933544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910953403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[0]R flush_numbers[0..12]: [79, 82, 83, 83, 83, 80, 75, 75, 82, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910970932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[0]R generation[0..12]: [42, 44, 45, 45, 45, 43, 38, 38, 44, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910985151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.910999341Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[1]R flush_numbers[0..12]: [79, 82, 83, 83, 83, 80, 75, 75, 82, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.91101329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[1]R generation[0..12]: [42, 44, 45, 45, 45, 43, 38, 38, 44, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.9110271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.9110413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[2]R flush_numbers[0..12]: [79, 82, 83, 83, 83, 80, 75, 75, 82, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911055139Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[2]R generation[0..12]: [42, 44, 45, 45, 45, 43, 38, 38, 44, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911068859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911082878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911095818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911109187Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Next flush: 84","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911122157Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911135206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"":"downstairs","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911148666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911162895Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"0","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911176665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"1","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911200874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"client":"2","":"downstairs","session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911214573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911227003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"e586d227-55ba-4235-98de-0becb368bcda is now active with session: d1eb27fd-62a3-4839-b061-048a58cd7369","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911239932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911251992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366,"session_id":"d1eb27fd-62a3-4839-b061-048a58cd7369"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:48.911264201Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 85 len 24576 data: 4 6 6 7 6 7 02/30 Write block 62 len 4096 data: 7 03/30 Read block 582 len 12288 04/30 Write block 125 len 32768 data: 4 5 5 4 3 3 4 4 05/30 Read block 140 len 36864 06/30 Read block 378 len 12288 07/30 Write block 54 len 28672 data: 5 3 4 5 7 7 7 08/30 Write block 384 len 8192 data: 5 5 09/30 Read block 501 len 12288 10/30 Write block 280 len 28672 data: 7 9 10 12 8 8 8 11/30 Write block 512 len 4096 data: 5 12/30 Read block 51 len 16384 13/30 Write block 290 len 40960 data: 5 6 5 4 2 3 3 3 3 3 14/30 Read block 177 len 12288 15/30 Read block 79 len 16384 16/30 Read block 289 len 28672 17/30 Flush 18/30 Write block 49 len 12288 data: 4 4 4 19/30 Read block 423 len 28672 20/30 Read block 585 len 20480 21/30 Read block 275 len 4096 22/30 Read block 417 len 4096 23/30 Read block 50 len 20480 24/30 Write block 119 len 40960 data: 6 5 4 3 2 3 5 6 6 5 25/30 Read block 426 len 4096 26/30 Write block 87 len 4096 data: 7 27/30 Read block 410 len 16384 28/30 Write block 392 len 40960 data: 6 5 6 6 5 5 6 6 5 4 29/30 Write block 121 len 12288 data: 5 4 3 30/30 Read block 462 len 36864 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 3 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 3 Done Done Done false 1029 Acked Read 9 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 12 12 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:48.934399256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1366} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 F F F 1 020-039 44 44 44 82 82 82 F F F 2 040-059 46 46 46 84 84 84 T T T 3 060-079 46 46 46 84 84 84 F F F 4 080-099 46 46 46 84 84 84 T T T 5 100-119 43 43 43 80 80 80 T T T 6 120-139 46 46 46 84 84 84 T T T 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 45 45 45 83 83 83 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 46 46 46 84 84 84 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 44 44 44 81 81 81 F F F 19 380-399 46 46 46 84 84 84 T T T 20 400-419 44 44 44 82 82 82 T T T 21 420-439 45 45 45 83 83 83 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 43 43 43 80 80 80 F F F 25 500-519 46 46 46 84 84 84 F F F 26 520-539 43 43 43 80 80 80 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 46, Max flush: 84 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 F F F 1 020-039 44 44 44 82 82 82 F F F 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 47 47 47 85 85 85 F F F 5 100-119 47 47 47 85 85 85 F F F 6 120-139 47 47 47 85 85 85 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 41 41 41 78 78 78 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 45 45 45 83 83 83 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 46 46 46 84 84 84 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 44 44 44 81 81 81 F F F 19 380-399 47 47 47 85 85 85 F F F 20 400-419 47 47 47 85 85 85 F F F 21 420-439 45 45 45 83 83 83 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 43 43 43 80 80 80 F F F 25 500-519 46 46 46 84 84 84 F F F 26 520-539 43 43 43 80 80 80 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 36 36 36 72 72 72 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 47, Max flush: 85 Begin loop 24 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.262372522Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.305577602Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.30618197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.306225769Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.306243608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.306261428Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} {"msg":"Crucible fdbf141e-8941-4aa3-88d3-4049870f7d54 has session id: 5aadcdc3-8409-4b77-89fb-1c953f351132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.306723881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Upstairs opts: Upstairs UUID: fdbf141e-8941-4aa3-88d3-4049870f7d54, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.3067676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Crucible stats registered with UUID: fdbf141e-8941-4aa3-88d3-4049870f7d54","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.306791868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336303913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336337212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} {"msg":"fdbf141e-8941-4aa3-88d3-4049870f7d54 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33640094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336434578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336468007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336486807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336504766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336521755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336538695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336555224Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336572353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336589103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_connection connected from Ok(127.0.0.1:37652)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33666922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"io task","client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_connection connected from Ok(127.0.0.1:52550)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.3369451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"io task","client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_connection connected from Ok(127.0.0.1:35233)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.336990749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"io task","client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.337392094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.337431233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.337546028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.337582087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.337690343Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.337735211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338003522Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338199025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33834155Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338515223Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338536853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338551092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338719716Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338739375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338753365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338859381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33887817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[0]R flush_numbers[0..12]: [79, 82, 85, 84, 85, 85, 85, 75, 82, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33889325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[0]R generation[0..12]: [42, 44, 47, 46, 47, 47, 47, 38, 44, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338907129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338921269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[1]R flush_numbers[0..12]: [79, 82, 85, 84, 85, 85, 85, 75, 82, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338938158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[1]R generation[0..12]: [42, 44, 47, 46, 47, 47, 47, 38, 44, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338952448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338966657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[2]R flush_numbers[0..12]: [79, 82, 85, 84, 85, 85, 85, 75, 82, 80, 78, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.338980477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[2]R generation[0..12]: [42, 44, 47, 46, 47, 47, 47, 38, 44, 43, 41, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339008376Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339024935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339038384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Generation requested: 48 >= found:48","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339051524Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Next flush: 86","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339064534Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339089463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"":"downstairs","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339114912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339136091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"0","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33915156Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"1","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.33916502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"client":"2","":"downstairs","session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339180859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339193339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"fdbf141e-8941-4aa3-88d3-4049870f7d54 is now active with session: 5aadcdc3-8409-4b77-89fb-1c953f351132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339206098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339218228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374,"session_id":"5aadcdc3-8409-4b77-89fb-1c953f351132"} {"msg"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:53.339234348Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 24576 data: 5 6 5 5 5 3 02/30 Write block 209 len 4096 data: 5 03/30 Write block 245 len 28672 data: 5 4 4 5 4 4 6 04/30 Read block 65 len 8192 05/30 Flush 06/30 Flush 07/30 Write block 410 len 28672 data: 5 4 4 4 4 3 3 08/30 Write block 111 len 28672 data: 4 4 6 5 6 6 6 09/30 Write block 565 len 32768 data: 7 7 8 7 6 6 5 4 10/30 Flush 11/30 Flush 12/30 Write block 94 len 12288 data: 6 6 4 13/30 Write block 114 len 28672 data: 6 7 7 7 6 7 6 14/30 Read block 583 len 24576 15/30 Write block 518 len 40960 data: 6 5 6 7 6 5 6 6 6 5 16/30 Flush 17/30 Write block 129 len 4096 data: 4 18/30 Flush 19/30 Write block 332 len 28672 data: 7 8 8 8 9 8 8 20/30 Write block 427 len 32768 data: 4 3 3 2 2 4 4 5 21/30 Read block 77 len 40960 22/30 Read block 426 len 4096 23/30 Write block 19 len 16384 data: 6 6 6 7 24/30 Read block 505 len 36864 25/30 Read block 205 len 20480 26/30 Read block 191 len 8192 27/30 Write block 484 len 8192 data: 6 6 28/30 Write block 89 len 32768 data: 7 8 5 5 6 7 7 5 29/30 Read block 340 len 36864 30/30 Write block 4 len 12288 data: 3 3 3 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 7 Done Done Done false 1019 Acked Write 8 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:53.365174401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1374} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 42 42 42 79 79 79 T T T 1 020-039 48 48 48 86 86 86 T T T 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 48 48 48 90 90 90 T T T 5 100-119 48 48 48 90 90 90 F F F 6 120-139 48 48 48 91 91 91 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 48 48 48 86 86 86 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 48 48 48 86 86 86 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 46 46 46 84 84 84 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 41 41 41 78 78 78 T T T 17 340-359 34 34 34 70 70 70 F F F 18 360-379 44 44 44 81 81 81 F F F 19 380-399 47 47 47 85 85 85 F F F 20 400-419 48 48 48 88 88 88 F F F 21 420-439 45 45 45 83 83 83 T T T 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 43 43 43 80 80 80 T T T 25 500-519 48 48 48 90 90 90 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 48 48 48 88 88 88 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 48, Max flush: 91 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 49 49 49 92 92 92 F F F 1 020-039 49 49 49 92 92 92 F F F 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 49 49 49 92 92 92 F F F 5 100-119 48 48 48 90 90 90 F F F 6 120-139 48 48 48 91 91 91 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 48 48 48 86 86 86 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 48 48 48 86 86 86 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 46 46 46 84 84 84 F F F 15 300-319 32 32 32 68 68 68 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 34 34 34 70 70 70 F F F 18 360-379 44 44 44 81 81 81 F F F 19 380-399 47 47 47 85 85 85 F F F 20 400-419 48 48 48 88 88 88 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 49 49 49 92 92 92 F F F 25 500-519 48 48 48 90 90 90 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 48 48 48 88 88 88 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 49, Max flush: 92 Begin loop 25 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.689330983Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.732213496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.732870062Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.732906081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.73292053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.73293566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} {"msg":"Crucible 0264f3ab-8188-4e8d-af09-93aab06301e3 has session id: a28a32d2-382d-4ed8-aa25-928ad018644d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.733351694Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Upstairs opts: Upstairs UUID: 0264f3ab-8188-4e8d-af09-93aab06301e3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.733392963Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Crucible stats registered with UUID: 0264f3ab-8188-4e8d-af09-93aab06301e3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.733410742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763336842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763370211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} {"msg":"0264f3ab-8188-4e8d-af09-93aab06301e3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763444558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763467917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763496026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763515786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763533135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763555634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763573514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763590213Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763607722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763624982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_connection connected from Ok(127.0.0.1:34515)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763760917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"io task","client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_connection connected from Ok(127.0.0.1:59315)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.763812365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"io task","client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_connection connected from Ok(127.0.0.1:57191)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.76393818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"io task","client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.764585737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.764622596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.764653475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.764683643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.764714982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.764742561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765258173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765376809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765409617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765818362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765848161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765862971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765965997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.765999226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766026865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766047174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766061284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[0]R flush_numbers[0..12]: [92, 92, 85, 84, 92, 90, 91, 75, 82, 80, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766075273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[0]R generation[0..12]: [49, 49, 47, 46, 49, 48, 48, 38, 44, 43, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766089223Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766103332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[1]R flush_numbers[0..12]: [92, 92, 85, 84, 92, 90, 91, 75, 82, 80, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766117122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[1]R generation[0..12]: [49, 49, 47, 46, 49, 48, 48, 38, 44, 43, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766130841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766148121Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[2]R flush_numbers[0..12]: [92, 92, 85, 84, 92, 90, 91, 75, 82, 80, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.76616244Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[2]R generation[0..12]: [49, 49, 47, 46, 49, 48, 48, 38, 44, 43, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.76617636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766190379Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"Wait for a query_work_queue command to finish before sending IO ":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766217968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766245007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766266536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766291605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"":"downstairs","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766317555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766342514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"0","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Transition from WaitQuorum to Active","v":0Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766371653Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"1","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766396692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"client":"2","":"downstairs","session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766411611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766424141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"0264f3ab-8188-4e8d-af09-93aab06301e3 is now active with session: a28a32d2-382d-4ed8-aa25-928ad018644d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.76643715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.76645209Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382,"session_id":"a28a32d2-382d-4ed8-aa25-928ad018644d"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:04:57.766464939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 8192 data: 6 7 02/30 Read block 90 len 32768 03/30 Read block 248 len 32768 04/30 Flush 05/30 Write block 394 len 32768 data: 7 7 6 6 7 7 6 5 06/30 Write block 119 len 8192 data: 8 7 07/30 Read block 318 len 32768 08/30 Read block 260 len 8192 09/30 Read block 537 len 12288 10/30 Read block 247 len 20480 11/30 Write block 386 len 12288 data: 5 3 5 12/30 Write block 575 len 16384 data: 3 3 4 4 13/30 Read block 451 len 4096 14/30 Read block 238 len 36864 15/30 Write block 569 len 12288 data: 7 7 6 16/30 Read block 3 len 36864 17/30 Write block 301 len 32768 data: 4 3 4 3 5 5 5 6 18/30 Read block 155 len 28672 19/30 Read block 563 len 36864 20/30 Flush 21/30 Write block 14 len 28672 data: 4 3 3 5 5 7 7 22/30 Read block 146 len 4096 23/30 Flush 24/30 Write block 359 len 20480 data: 4 5 6 6 8 25/30 Write block 130 len 40960 data: 4 5 5 6 8 7 7 6 3 3 26/30 Flush 27/30 Write block 511 len 4096 data: 7 28/30 Read block 259 len 24576 29/30 Read block 547 len 20480 30/30 Read block 29 len 20480 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 4 4 3 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:04:57.791852212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1382} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 F F F 1 020-039 50 50 50 95 95 95 F F F 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 49 49 49 92 92 92 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 50 50 50 96 96 96 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 48 48 48 86 86 86 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 48 48 48 86 86 86 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 50 50 50 93 93 93 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 50 50 50 96 96 96 F F F 18 360-379 50 50 50 96 96 96 F F F 19 380-399 50 50 50 94 94 94 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 49 49 49 92 92 92 F F F 25 500-519 48 48 48 90 90 90 T T T 26 520-539 48 48 48 90 90 90 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 50 50 50 94 94 94 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 50, Max flush: 96 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 50 50 50 95 95 95 F F F 1 020-039 50 50 50 95 95 95 F F F 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 49 49 49 92 92 92 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 50 50 50 96 96 96 F F F 7 140-159 38 38 38 75 75 75 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 43 43 43 80 80 80 F F F 10 200-219 48 48 48 86 86 86 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 48 48 48 86 86 86 F F F 13 260-279 38 38 38 75 75 75 F F F 14 280-299 50 50 50 93 93 93 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 50 50 50 96 96 96 F F F 18 360-379 50 50 50 96 96 96 F F F 19 380-399 50 50 50 94 94 94 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 43 43 43 80 80 80 F F F 23 460-479 44 44 44 82 82 82 F F F 24 480-499 49 49 49 92 92 92 F F F 25 500-519 51 51 51 97 97 97 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 43 43 43 80 80 80 F F F 28 560-579 50 50 50 94 94 94 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 51, Max flush: 97 Begin loop 26 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.051259842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.095715287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.09646682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.096504779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.096519368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.096534038Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} {"msg":"Crucible c0d46b47-7d28-4a7f-886b-209a6323d950 has session id: 0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.096967562Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Upstairs opts: Upstairs UUID: c0d46b47-7d28-4a7f-886b-209a6323d950, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.09701111Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Crucible stats registered with UUID: c0d46b47-7d28-4a7f-886b-209a6323d950","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.09703528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127437102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127468961Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} {"msg":"c0d46b47-7d28-4a7f-886b-209a6323d950 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127537988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127560018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127588277Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127606866Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127638185Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127671264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127705282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127734732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127753421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.12777074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:33431)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127796849Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"io task","client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:48416)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.127913735Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"io task","client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62747)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.12805791Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"io task","client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.128574071Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.12860088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.12861584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.128629599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.128643219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.128656848Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.129272066Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.129410991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.12944204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.129908843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.129938212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.129953521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130042218Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130066117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130080247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130092906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130106536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[0]R flush_numbers[0..12]: [95, 95, 85, 84, 92, 94, 96, 75, 82, 80, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130120495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[0]R generation[0..12]: [50, 50, 47, 46, 49, 50, 50, 38, 44, 43, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130134435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130148614Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[1]R flush_numbers[0..12]: [95, 95, 85, 84, 92, 94, 96, 75, 82, 80, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130162494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[1]R generation[0..12]: [50, 50, 47, 46, 49, 50, 50, 38, 44, 43, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130176023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130189863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[2]R flush_numbers[0..12]: [95, 95, 85, 84, 92, 94, 96, 75, 82, 80, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130203582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[2]R generation[0..12]: [50, 50, 47, 46, 49, 50, 50, 38, 44, 43, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130217352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130231501Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130244611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.13025973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Next flush: 98","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:05:02.13027314Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130292389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"":"downstairs","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130305689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130317808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"0","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130331048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"1","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130344197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"client":"2","":"downstairs","session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130357537Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130369836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"c0d46b47-7d28-4a7f-886b-209a6323d950 is now active with session: 0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130382546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130394685Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390,"session_id":"0fb2f9d1-a1e2-472a-9205-0dab7be4a3d3"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:02.130407005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Flush 03/30 Write block 382 len 16384 data: 5 5 6 6 04/30 Flush 05/30 Flush 06/30 Write block 366 len 8192 data: 4 4 07/30 Write block 151 len 8192 data: 2 3 08/30 Write block 180 len 4096 data: 5 09/30 Read block 352 len 40960 10/30 Read block 336 len 20480 11/30 Write block 377 len 20480 data: 4 4 5 5 5 12/30 Write block 545 len 32768 data: 5 8 8 8 8 7 7 8 13/30 Read block 135 len 40960 14/30 Read block 436 len 8192 15/30 Write block 186 len 24576 data: 5 5 3 4 4 4 16/30 Read block 345 len 20480 17/30 Read block 124 len 36864 18/30 Read block 161 len 32768 19/30 Read block 100 len 4096 20/30 Write block 360 len 4096 data: 6 21/30 Write block 13 len 4096 data: 4 22/30 Flush 23/30 Write block 458 len 28672 data: 7 7 5 7 6 6 4 24/30 Flush 25/30 Write block 120 len 28672 data: 8 6 5 4 4 6 7 26/30 Flush 27/30 Write block 549 len 8192 data: 9 8 28/30 Write block 271 len 40960 data: 7 7 5 6 5 5 5 5 5 8 29/30 Write block 185 len 32768 data: 5 6 6 4 5 5 5 5 30/30 Write block 252 len 24576 data: 6 6 5 6 6 3 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Sent Done Sent false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 3 4 11 Done 0 1 0 1 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:02.158226681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1390} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 F F F 1 020-039 50 50 50 95 95 95 F F F 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 49 49 49 92 92 92 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 52 52 52 104 104 104 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 52 52 52 102 102 102 T T T 10 200-219 48 48 48 86 86 86 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 48 48 48 86 86 86 T T T 13 260-279 38 38 38 75 75 75 T T T 14 280-299 50 50 50 93 93 93 T T T 15 300-319 50 50 50 94 94 94 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 50 50 50 96 96 96 F F F 18 360-379 52 52 52 102 102 102 F F F 19 380-399 52 52 52 102 102 102 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 49 49 49 92 92 92 F F F 25 500-519 51 51 51 97 97 97 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 52 52 52 102 102 102 T T T 28 560-579 50 50 50 94 94 94 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 52, Max flush: 104 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 F F F 1 020-039 50 50 50 95 95 95 F F F 2 040-059 47 47 47 85 85 85 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 49 49 49 92 92 92 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 52 52 52 104 104 104 F F F 7 140-159 52 52 52 102 102 102 F F F 8 160-179 44 44 44 82 82 82 F F F 9 180-199 53 53 53 105 105 105 F F F 10 200-219 48 48 48 86 86 86 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 53 53 53 105 105 105 F F F 13 260-279 53 53 53 105 105 105 F F F 14 280-299 53 53 53 105 105 105 F F F 15 300-319 50 50 50 94 94 94 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 50 50 50 96 96 96 F F F 18 360-379 52 52 52 102 102 102 F F F 19 380-399 52 52 52 102 102 102 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 49 49 49 92 92 92 F F F 25 500-519 51 51 51 97 97 97 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 53 53 53 105 105 105 F F F 28 560-579 50 50 50 94 94 94 F F F 29 580-599 44 44 44 82 82 82 F F F Max gen: 53, Max flush: 105 Begin loop 27 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.469538157Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:06.511488762Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.511888498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.542717695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543271155Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543314794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543331953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543349852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} {"msg":"Crucible a71d8707-3843-4454-91dd-945e1262ddbb has session id: 290ed04c-0448-4e6e-9b6a-bece2678efe9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543744268Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Upstairs opts: Upstairs UUID: a71d8707-3843-4454-91dd-945e1262ddbb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543784366Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Crucible stats registered with UUID: a71d8707-3843-4454-91dd-945e1262ddbb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.543813976Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.574964141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.57500055Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} {"msg":"a71d8707-3843-4454-91dd-945e1262ddbb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575079207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575102966Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575132345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575150404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575167394Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575184003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575240181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575293419Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575325828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575353487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37628)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575380546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"io task","client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_connection connected from Ok(127.0.0.1:35149)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575409015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"io task","client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_connection connected from Ok(127.0.0.1:45120)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.575454643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"io task","client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.57609406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576133309Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576267174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576299283Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576431978Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576469847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576849803Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.576874502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577156212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577432742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577457391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577472421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577560467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577580566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577594436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577910645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577937564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[0]R flush_numbers[0..12]: [102, 95, 85, 84, 92, 94, 104, 102, 82, 105, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577962493Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[0]R generation[0..12]: [52, 50, 47, 46, 49, 50, 52, 52, 44, 53, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.577984652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578020901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[1]R flush_numbers[0..12]: [102, 95, 85, 84, 92, 94, 104, 102, 82, 105, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.57804274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[1]R generation[0..12]: [52, 50, 47, 46, 49, 50, 52, 52, 44, 53, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578063399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578081199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[2]R flush_numbers[0..12]: [102, 95, 85, 84, 92, 94, 104, 102, 82, 105, 86, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578107027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"[2]R generation[0..12]: [52, 50, 47, 46, 49, 50, 52, 52, 44, 53, 48, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578134167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578170895Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578200074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578226683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Next flush: 106","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578254332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578273252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"":"downstairs","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578290001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.57831234Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"0","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578338949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"1","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578366128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"client":"2","":"downstairs","session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578382988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578397387Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"a71d8707-3843-4454-91dd-945e1262ddbb is now active with session: 290ed04c-0448-4e6e-9b6a-bece2678efe9","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:05:06.578422626Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578447435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398,"session_id":"290ed04c-0448-4e6e-9b6a-bece2678efe9"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:06.578468954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 32768 data: 7 7 7 6 7 7 4 3 02/30 Read block 499 len 40960 03/30 Read block 517 len 40960 04/30 Read block 533 len 12288 05/30 Write block 34 len 32768 data: 6 5 5 7 7 7 6 6 06/30 Write block 189 len 4096 data: 6 07/30 Read block 184 len 24576 08/30 Flush 09/30 Write block 272 len 36864 data: 8 6 7 6 6 6 6 6 9 10/30 Read block 429 len 28672 11/30 Write block 486 len 4096 data: 6 12/30 Read block 218 len 4096 13/30 Write block 274 len 24576 data: 8 7 7 7 7 7 14/30 Write block 582 len 36864 data: 4 7 8 9 9 10 10 9 8 15/30 Read block 295 len 8192 16/30 Read block 388 len 20480 17/30 Write block 159 len 8192 data: 4 5 18/30 Write block 90 len 20480 data: 9 6 6 7 8 19/30 Read block 493 len 24576 20/30 Write block 371 len 24576 data: 7 8 7 6 6 4 21/30 Write block 370 len 36864 data: 8 8 9 8 7 7 5 5 5 22/30 Read block 177 len 28672 23/30 Read block 3 len 4096 24/30 Read block 520 len 12288 25/30 Write block 499 len 32768 data: 7 7 6 6 6 7 8 7 26/30 Read block 367 len 4096 27/30 Write block 298 len 24576 data: 4 4 3 5 4 5 28/30 Write block 205 len 28672 data: 6 6 5 3 6 5 5 29/30 Write block 345 len 20480 data: 4 4 4 5 5 30/30 Write block 308 len 28672 data: 7 5 5 5 4 3 3 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:22 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1008 Acked Write 9 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 1 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Write 6 Done Done Done false 1013 Acked Write 9 Done Done Done false 1014 Acked Read 2 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 6 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Write 6 Done Done Sent false 1027 Acked Write 7 Done Done Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 4 8 Done 20 20 18 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1007 1007 1007 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:06.600579896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1398} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 F F F 1 020-039 54 54 54 106 106 106 F F F 2 040-059 54 54 54 106 106 106 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 49 49 49 92 92 92 T T T 5 100-119 50 50 50 94 94 94 F F F 6 120-139 52 52 52 104 104 104 F F F 7 140-159 52 52 52 102 102 102 T T T 8 160-179 44 44 44 82 82 82 T T T 9 180-199 54 54 54 106 106 106 F F F 10 200-219 48 48 48 86 86 86 T T T 11 220-239 43 43 43 80 80 80 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 53 53 53 105 105 105 T T T 14 280-299 53 53 53 105 105 105 T T T 15 300-319 50 50 50 94 94 94 T T T 16 320-339 49 49 49 92 92 92 F F F 17 340-359 50 50 50 96 96 96 T T T 18 360-379 52 52 52 102 102 102 T T T 19 380-399 52 52 52 102 102 102 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 49 49 49 92 92 92 T T T 25 500-519 51 51 51 97 97 97 T T T 26 520-539 48 48 48 90 90 90 F F F 27 540-559 53 53 53 105 105 105 F F F 28 560-579 50 50 50 94 94 94 F F F 29 580-599 44 44 44 82 82 82 T T T Max gen: 54, Max flush: 106 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 F F F 1 020-039 54 54 54 106 106 106 F F F 2 040-059 54 54 54 106 106 106 F F F 3 060-079 46 46 46 84 84 84 F F F 4 080-099 55 55 55 107 107 107 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 52 52 52 104 104 104 F F F 7 140-159 55 55 55 107 107 107 F F F 8 160-179 55 55 55 107 107 107 F F F 9 180-199 54 54 54 106 106 106 F F F 10 200-219 55 55 55 107 107 107 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 55 55 55 107 107 107 F F F 18 360-379 55 55 55 107 107 107 F F F 19 380-399 52 52 52 102 102 102 F F F 20 400-419 50 50 50 94 94 94 F F F 21 420-439 49 49 49 92 92 92 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 55 55 55 107 107 107 F F F 25 500-519 55 55 55 107 107 107 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 53 53 53 105 105 105 F F F 28 560-579 50 50 50 94 94 94 F F F 29 580-599 55 55 55 107 107 107 F F F Max gen: 55, Max flush: 107 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.035371304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.078264726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.078871264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.078907633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.078922502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.078938622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} {"msg":"Crucible 4f11b0ff-2898-466e-b4e2-d222eec88d46 has session id: 57c064ba-2a2e-437c-9f88-99c80df326f5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.079424894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Upstairs opts: Upstairs UUID: 4f11b0ff-2898-466e-b4e2-d222eec88d46, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.079468743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Crucible stats registered with UUID: 4f11b0ff-2898-466e-b4e2-d222eec88d46","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.079492662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.108952348Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.108985857Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} {"msg":"4f11b0ff-2898-466e-b4e2-d222eec88d46 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109036755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109057495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109083763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109101463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109118622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109147241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.1091808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109213259Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109249917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109284636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:41895)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109318195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"io task","client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42188)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109352204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"io task","client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_connection connected from Ok(127.0.0.1:46396)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.109591535Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"io task","client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110135805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110177054Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110340438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110382007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110416126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110447564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.110779162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.1111197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111162219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111347522Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.1113857Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.11140257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111685429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111715599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111730828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111743938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111762197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[0]R flush_numbers[0..12]: [102, 106, 106, 84, 107, 94, 104, 107, 107, 106, 107, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111776786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[0]R generation[0..12]: [52, 54, 54, 46, 55, 50, 52, 55, 55, 54, 55, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111790826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111805045Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[1]R flush_numbers[0..12]: [102, 106, 106, 84, 107, 94, 104, 107, 107, 106, 107, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111818975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[1]R generation[0..12]: [52, 54, 54, 46, 55, 50, 52, 55, 55, 54, 55, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111832714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111846644Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[2]R flush_numbers[0..12]: [102, 106, 106, 84, 107, 94, 104, 107, 107, 106, 107, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111860223Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[2]R generation[0..12]: [52, 54, 54, 46, 55, 50, 52, 55, 55, 54, 55, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111874023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111887912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111901022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111929101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.11195673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111975279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"":"downstairs","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.111992158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112029947Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"0","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112054806Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"1","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112069286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"client":"2","":"downstairs","session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112082775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112098095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"4f11b0ff-2898-466e-b4e2-d222eec88d46 is now active with session: 57c064ba-2a2e-437c-9f88-99c80df326f5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112122954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112142553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406,"session_id":"57c064ba-2a2e-437c-9f88-99c80df326f5"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:11.112155643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 28672 data: 9 8 8 8 7 5 4 02/30 Flush 03/30 Flush 04/30 Write block 130 len 12288 data: 5 6 6 05/30 Read block 403 len 4096 06/30 Read block 295 len 24576 07/30 Read block 171 len 20480 08/30 Read block 54 len 28672 09/30 Read block 327 len 28672 10/30 Write block 172 len 24576 data: 4 3 3 4 4 4 11/30 Read block 486 len 36864 12/30 Flush 13/30 Write block 595 len 20480 data: 3 3 3 3 3 14/30 Write block 121 len 36864 data: 7 6 5 5 7 8 7 6 5 15/30 Flush 16/30 Write block 204 len 28672 data: 6 7 7 6 4 7 6 17/30 Write block 202 len 32768 data: 5 6 7 8 8 7 5 8 18/30 Write block 420 len 24576 data: 4 4 3 6 4 5 19/30 Read block 56 len 24576 20/30 Write block 216 len 12288 data: 6 6 6 21/30 Write block 63 len 40960 data: 7 5 5 5 4 3 4 4 4 3 22/30 Read block 485 len 36864 23/30 Write block 52 len 16384 data: 4 5 6 4 24/30 Write block 197 len 4096 data: 4 25/30 Read block 369 len 4096 26/30 Write block 343 len 8192 data: 5 5 27/30 Write block 362 len 36864 data: 7 9 8 5 5 5 5 7 9 28/30 Write block 190 len 20480 data: 6 6 6 6 5 29/30 Read block 181 len 32768 30/30 Write block 409 len 40960 data: 6 6 5 5 5 5 4 4 3 4 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 7 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 1 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Write 2 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 14 14 14 42 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:11.13827646Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1406} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 F F F 1 020-039 54 54 54 106 106 106 F F F 2 040-059 54 54 54 106 106 106 T T T 3 060-079 46 46 46 84 84 84 T T T 4 080-099 55 55 55 107 107 107 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 56 56 56 111 111 111 F F F 7 140-159 55 55 55 107 107 107 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 54 54 54 106 106 106 T T T 10 200-219 55 55 55 107 107 107 T T T 11 220-239 43 43 43 80 80 80 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 55 55 55 107 107 107 T T T 18 360-379 55 55 55 107 107 107 T T T 19 380-399 52 52 52 102 102 102 F F F 20 400-419 50 50 50 94 94 94 T T T 21 420-439 49 49 49 92 92 92 T T T 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 55 55 55 107 107 107 F F F 25 500-519 55 55 55 107 107 107 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 53 53 53 105 105 105 F F F 28 560-579 56 56 56 108 108 108 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 56, Max flush: 111 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 F F F 1 020-039 54 54 54 106 106 106 F F F 2 040-059 57 57 57 112 112 112 F F F 3 060-079 57 57 57 112 112 112 F F F 4 080-099 55 55 55 107 107 107 F F F 5 100-119 50 50 50 94 94 94 F F F 6 120-139 56 56 56 111 111 111 F F F 7 140-159 55 55 55 107 107 107 F F F 8 160-179 56 56 56 110 110 110 F F F 9 180-199 57 57 57 112 112 112 F F F 10 200-219 57 57 57 112 112 112 F F F 11 220-239 43 43 43 80 80 80 F F F 12 240-259 54 54 54 106 106 106 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 49 49 49 92 92 92 F F F 17 340-359 57 57 57 112 112 112 F F F 18 360-379 57 57 57 112 112 112 F F F 19 380-399 52 52 52 102 102 102 F F F 20 400-419 57 57 57 112 112 112 F F F 21 420-439 57 57 57 112 112 112 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 55 55 55 107 107 107 F F F 25 500-519 55 55 55 107 107 107 F F F 26 520-539 48 48 48 90 90 90 F F F 27 540-559 53 53 53 105 105 105 F F F 28 560-579 56 56 56 108 108 108 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 57, Max flush: 112 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.486979836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:15.53062777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.531036836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.561912391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.562589747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.562626045Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.562640305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.562654934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} {"msg":"Crucible 9b2eeb67-2e4a-4c2f-9f70-278dafb3bd23 has session id: 54fff7f4-5f5e-4a52-ba80-4ae268af7c52","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.563145087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Upstairs opts: Upstairs UUID: 9b2eeb67-2e4a-4c2f-9f70-278dafb3bd23, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.563179146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Crucible stats registered with UUID: 9b2eeb67-2e4a-4c2f-9f70-278dafb3bd23","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.563198935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592736088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592766367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} {"msg":"9b2eeb67-2e4a-4c2f-9f70-278dafb3bd23 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592833705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592856324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592883803Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592901302Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.592926242Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.59295932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593014998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593054787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593091096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593125445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_connection connected from Ok(127.0.0.1:34160)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593158903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"io task","client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_connection connected from Ok(127.0.0.1:50695)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593192342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"io task","client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_connection connected from Ok(127.0.0.1:38779)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593277389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"io task","client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.593962824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594005453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594037441Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.59406821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594098599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594128938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594555573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594671079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.594813044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595047285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595087124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595117153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.59519919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595228469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595244638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595265557Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595283147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[0]R flush_numbers[0..12]: [102, 106, 112, 112, 107, 94, 111, 107, 110, 112, 112, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595309766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[0]R generation[0..12]: [52, 54, 57, 57, 55, 50, 56, 55, 56, 57, 57, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595324345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595338874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[1]R flush_numbers[0..12]: [102, 106, 112, 112, 107, 94, 111, 107, 110, 112, 112, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595353174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[1]R generation[0..12]: [52, 54, 57, 57, 55, 50, 56, 55, 56, 57, 57, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595366773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595381073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[2]R flush_numbers[0..12]: [102, 106, 112, 112, 107, 94, 111, 107, 110, 112, 112, 80]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595394983Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[2]R generation[0..12]: [52, 54, 57, 57, 55, 50, 56, 55, 56, 57, 57, 43]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595408742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595422711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595435821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595449241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Next flush: 113","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.59546247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.59547564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"":"downstairs","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595491809Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595512828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"0","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595526128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"1","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595539237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"client":"2","":"downstairs","session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595552557Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595564986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"9b2eeb67-2e4a-4c2f-9f70-278dafb3bd23 is now active with session: 54fff7f4-5f5e-4a52-ba80-4ae268af7c52","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595577856Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595590096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414,"session_id":"54fff7f4-5f5e-4a52-ba80-4ae268af7c52"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:15.595602575Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 251 len 32768 data: 8 8 8 7 8 8 5 4 02/30 Write block 150 len 4096 data: 2 03/30 Write block 237 len 24576 data: 3 4 6 6 4 4 04/30 Write block 105 len 32768 data: 4 3 3 3 4 4 5 5 05/30 Write block 55 len 32768 data: 5 5 6 8 8 8 5 8 06/30 Read block 129 len 4096 07/30 Read block 161 len 36864 08/30 Read block 370 len 24576 09/30 Read block 6 len 4096 10/30 Write block 155 len 28672 data: 4 4 4 4 5 6 4 11/30 Write block 156 len 8192 data: 5 5 12/30 Write block 331 len 20480 data: 7 8 9 9 9 13/30 Read block 543 len 24576 14/30 Flush 15/30 Write block 13 len 40960 data: 5 5 4 4 6 6 8 8 7 8 16/30 Read block 339 len 20480 17/30 Read block 503 len 8192 18/30 Write block 389 len 20480 data: 5 5 5 7 6 19/30 Write block 216 len 8192 data: 7 7 20/30 Write block 399 len 16384 data: 8 7 6 4 21/30 Read block 325 len 24576 22/30 Write block 495 len 16384 data: 7 7 6 5 23/30 Write block 548 len 24576 data: 9 10 9 8 9 8 24/30 Read block 564 len 12288 25/30 Write block 14 len 36864 data: 6 5 5 7 7 9 9 8 9 26/30 Write block 516 len 32768 data: 4 6 7 6 7 8 7 6 27/30 Write block 379 len 40960 data: 6 6 6 6 6 7 7 6 4 6 28/30 Write block 423 len 4096 data: 7 29/30 Read block 267 len 36864 30/30 Write block 28 len 28672 data: 6 4 4 5 5 5 7 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Read 5 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Write 5 Done Done Done false 1018 Acked Write 2 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:15.615920362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1414} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 52 52 52 102 102 102 T T T 1 020-039 54 54 54 106 106 106 T T T 2 040-059 58 58 58 113 113 113 F F F 3 060-079 58 58 58 113 113 113 F F F 4 080-099 55 55 55 107 107 107 F F F 5 100-119 58 58 58 113 113 113 F F F 6 120-139 56 56 56 111 111 111 F F F 7 140-159 58 58 58 113 113 113 F F F 8 160-179 58 58 58 113 113 113 F F F 9 180-199 57 57 57 112 112 112 F F F 10 200-219 57 57 57 112 112 112 T T T 11 220-239 58 58 58 113 113 113 F F F 12 240-259 58 58 58 113 113 113 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 57 57 57 112 112 112 F F F 18 360-379 57 57 57 112 112 112 T T T 19 380-399 52 52 52 102 102 102 T T T 20 400-419 57 57 57 112 112 112 T T T 21 420-439 57 57 57 112 112 112 T T T 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 55 55 55 107 107 107 T T T 25 500-519 55 55 55 107 107 107 T T T 26 520-539 48 48 48 90 90 90 T T T 27 540-559 53 53 53 105 105 105 T T T 28 560-579 56 56 56 108 108 108 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 58, Max flush: 113 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 114 114 114 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 58 58 58 113 113 113 F F F 4 080-099 55 55 55 107 107 107 F F F 5 100-119 58 58 58 113 113 113 F F F 6 120-139 56 56 56 111 111 111 F F F 7 140-159 58 58 58 113 113 113 F F F 8 160-179 58 58 58 113 113 113 F F F 9 180-199 57 57 57 112 112 112 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 58 58 58 113 113 113 F F F 13 260-279 55 55 55 107 107 107 F F F 14 280-299 55 55 55 107 107 107 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 57 57 57 112 112 112 F F F 18 360-379 59 59 59 114 114 114 F F F 19 380-399 59 59 59 114 114 114 F F F 20 400-419 59 59 59 114 114 114 F F F 21 420-439 59 59 59 114 114 114 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 59 59 59 114 114 114 F F F 25 500-519 59 59 59 114 114 114 F F F 26 520-539 59 59 59 114 114 114 F F F 27 540-559 59 59 59 114 114 114 F F F 28 560-579 56 56 56 108 108 108 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 59, Max flush: 114 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.035697332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.080138617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.080749655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.080785414Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.080800053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.080814833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} {"msg":"Crucible e9ec8c1e-b598-49ed-be06-0b21a25da9dc has session id: c5565ba2-50a6-4d96-941c-97b24f089c64","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.081238708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Upstairs opts: Upstairs UUID: e9ec8c1e-b598-49ed-be06-0b21a25da9dc, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.081275666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Crucible stats registered with UUID: e9ec8c1e-b598-49ed-be06-0b21a25da9dc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.081292146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.111819074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.111850543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} {"msg":"e9ec8c1e-b598-49ed-be06-0b21a25da9dc active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.11192443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.111946299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.111974588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.111992307Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112009467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112026766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112044106Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112060775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112077944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112094894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_connection connected from Ok(127.0.0.1:46507)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112176781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"io task","client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_connection connected from Ok(127.0.0.1:59135)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112218609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"io task","client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_connection connected from Ok(127.0.0.1:34629)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.112423832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"io task","client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113049059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113080288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113095338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113109317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113122837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113136396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113692766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113832261Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.113893589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114269675Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114298234Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114314654Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114357002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114374421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114388281Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.1144031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.11442068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[0]R flush_numbers[0..12]: [114, 114, 113, 113, 107, 113, 111, 113, 113, 112, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114447789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[0]R generation[0..12]: [59, 59, 58, 58, 55, 58, 56, 58, 58, 57, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114473078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114487897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[1]R flush_numbers[0..12]: [114, 114, 113, 113, 107, 113, 111, 113, 113, 112, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114501827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[1]R generation[0..12]: [59, 59, 58, 58, 55, 58, 56, 58, 58, 57, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114515387Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114529206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[2]R flush_numbers[0..12]: [114, 114, 113, 113, 107, 113, 111, 113, 113, 112, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114543066Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[2]R generation[0..12]: [59, 59, 58, 58, 55, 58, 56, 58, 58, 57, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114556515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114570554Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114583724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114597104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Next flush: 115","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114610833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114623672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"":"downstairs","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.1146903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114718779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"0","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114734608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"1","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114748848Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"client":"2","":"downstairs","session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114762498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114775127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"e9ec8c1e-b598-49ed-be06-0b21a25da9dc is now active with session: c5565ba2-50a6-4d96-941c-97b24f089c64","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114788126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114800316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422,"session_id":"c5565ba2-50a6-4d96-941c-97b24f089c64"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:20.114812926Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 70 len 40960 data: 5 5 4 2 3 4 5 3 3 3 02/30 Read block 349 len 32768 03/30 Flush 04/30 Write block 286 len 16384 data: 9 8 7 5 05/30 Read block 476 len 16384 06/30 Read block 2 len 32768 07/30 Read block 124 len 4096 08/30 Read block 62 len 36864 09/30 Write block 290 len 12288 data: 7 8 6 10/30 Read block 154 len 32768 11/30 Read block 349 len 16384 12/30 Write block 393 len 32768 data: 7 8 8 7 7 8 9 8 13/30 Read block 43 len 40960 14/30 Read block 518 len 12288 15/30 Flush 16/30 Write block 101 len 36864 data: 3 4 4 5 5 4 4 4 5 17/30 Read block 409 len 36864 18/30 Write block 110 len 4096 data: 5 19/30 Flush 20/30 Write block 267 len 4096 data: 7 21/30 Write block 163 len 12288 data: 4 5 6 22/30 Write block 569 len 12288 data: 9 9 8 23/30 Write block 194 len 16384 data: 6 3 3 5 24/30 Flush 25/30 Write block 530 len 24576 data: 5 6 6 5 5 4 26/30 Write block 571 len 12288 data: 9 6 5 27/30 Write block 388 len 8192 data: 7 6 28/30 Write block 128 len 4096 data: 7 29/30 Read block 545 len 8192 30/30 Read block 322 len 4096 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 6 Done Done Sent false 1025 Acked Write 3 Sent Done Sent false 1026 Acked Write 2 Sent Done Sent false 1027 Acked Write 1 Sent Done Sent false 1028 Acked Read 2 Sent Done Sent false 1029 Acked Read 1 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 5 0 6 11 Done 1 6 0 7 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:20.139554123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1422} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 114 114 114 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 60 60 60 115 115 115 F F F 4 080-099 55 55 55 107 107 107 F F F 5 100-119 60 60 60 117 117 117 F F F 6 120-139 56 56 56 111 111 111 T T T 7 140-159 58 58 58 113 113 113 F F F 8 160-179 60 60 60 118 118 118 F F F 9 180-199 60 60 60 118 118 118 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 58 58 58 113 113 113 F F F 13 260-279 60 60 60 118 118 118 F F F 14 280-299 60 60 60 116 116 116 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 57 57 57 112 112 112 F F F 18 360-379 59 59 59 114 114 114 F F F 19 380-399 60 60 60 116 116 116 T T T 20 400-419 60 60 60 116 116 116 F F F 21 420-439 59 59 59 114 114 114 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 59 59 59 114 114 114 F F F 25 500-519 59 59 59 114 114 114 F F F 26 520-539 59 59 59 114 114 114 T T T 27 540-559 59 59 59 114 114 114 F F F 28 560-579 60 60 60 118 118 118 T T T 29 580-599 56 56 56 111 111 111 F F F Max gen: 60, Max flush: 118 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 114 114 114 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 60 60 60 115 115 115 F F F 4 080-099 55 55 55 107 107 107 F F F 5 100-119 60 60 60 117 117 117 F F F 6 120-139 61 61 61 119 119 119 F F F 7 140-159 58 58 58 113 113 113 F F F 8 160-179 60 60 60 118 118 118 F F F 9 180-199 60 60 60 118 118 118 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 58 58 58 113 113 113 F F F 13 260-279 60 60 60 118 118 118 F F F 14 280-299 60 60 60 116 116 116 F F F 15 300-319 55 55 55 107 107 107 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 57 57 57 112 112 112 F F F 18 360-379 59 59 59 114 114 114 F F F 19 380-399 61 61 61 119 119 119 F F F 20 400-419 60 60 60 116 116 116 F F F 21 420-439 59 59 59 114 114 114 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 59 59 59 114 114 114 F F F 25 500-519 59 59 59 114 114 114 F F F 26 520-539 61 61 61 119 119 119 F F F 27 540-559 59 59 59 114 114 114 F F F 28 560-579 61 61 61 119 119 119 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 61, Max flush: 119 Begin loop 31 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.440771473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.483999673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.484588662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.48462622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.48464087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.484655629Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} {"msg":"Crucible 7f06e49d-3e83-4395-9895-6409b842253e has session id: b88faeea-4b78-4892-a90c-3e0228c7a79b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.485134642Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Upstairs opts: Upstairs UUID: 7f06e49d-3e83-4395-9895-6409b842253e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.48518842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Crucible stats registered with UUID: 7f06e49d-3e83-4395-9895-6409b842253e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.485218299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.514977605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515036673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} {"msg":"7f06e49d-3e83-4395-9895-6409b842253e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515054602Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515071241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.51509827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515121519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515153688Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515188627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515223536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515257754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515291683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515323782Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_connection connected from Ok(127.0.0.1:42847)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515434808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"io task","client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_connection connected from Ok(127.0.0.1:40902)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515479117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"io task","client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_connection connected from Ok(127.0.0.1:44562)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.515502826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"io task","client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516177391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.51621177Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.51622911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516276588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516341595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516385764Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516886436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516926664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.516960053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517483024Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517521843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517538942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517552062Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517566171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517579831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.51759243Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.51760605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[0]R flush_numbers[0..12]: [114, 114, 113, 115, 107, 117, 119, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517619959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[0]R generation[0..12]: [59, 59, 58, 60, 55, 60, 61, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517633879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517648068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[1]R flush_numbers[0..12]: [114, 114, 113, 115, 107, 117, 119, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517665388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[1]R generation[0..12]: [59, 59, 58, 60, 55, 60, 61, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517679517Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517693547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[2]R flush_numbers[0..12]: [114, 114, 113, 115, 107, 117, 119, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517707546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[2]R generation[0..12]: [59, 59, 58, 60, 55, 60, 61, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517721116Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517734985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517760654Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517787453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Next flush: 120","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517816332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517843881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"":"downstairs","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.51787326Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517901469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"0","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517934478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"1","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517961477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"client":"2","":"downstairs","session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517979256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.517992066Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"7f06e49d-3e83-4395-9895-6409b842253e is now active with session: b88faeea-4b78-4892-a90c-3e0228c7a79b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.518004985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.518017125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430,"session_id":"b88faeea-4b78-4892-a90c-3e0228c7a79b"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:24.518029545Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 379 len 32768 data: 7 7 7 7 7 8 8 7 02/30 Flush 03/30 Write block 255 len 36864 data: 9 9 6 5 3 3 2 2 2 04/30 Read block 485 len 8192 05/30 Write block 92 len 28672 data: 7 8 9 8 6 5 3 06/30 Write block 310 len 40960 data: 6 6 5 4 4 2 7 8 8 8 07/30 Write block 362 len 40960 data: 8 10 9 6 6 6 6 8 10 9 08/30 Read block 264 len 36864 09/30 Read block 298 len 36864 10/30 Flush 11/30 Write block 130 len 4096 data: 6 12/30 Read block 238 len 8192 13/30 Write block 380 len 40960 data: 8 8 8 8 9 9 8 5 8 7 14/30 Read block 482 len 4096 15/30 Read block 535 len 24576 16/30 Write block 371 len 28672 data: 10 10 9 8 8 6 6 17/30 Read block 131 len 32768 18/30 Write block 505 len 24576 data: 9 8 5 5 5 7 19/30 Read block 461 len 36864 20/30 Read block 504 len 24576 21/30 Write block 277 len 8192 data: 8 8 22/30 Read block 239 len 24576 23/30 Read block 373 len 40960 24/30 Write block 353 len 32768 data: 3 4 3 3 3 4 5 7 25/30 Read block 41 len 28672 26/30 Write block 6 len 32768 data: 4 3 2 3 2 2 2 6 27/30 Read block 188 len 16384 28/30 Write block 553 len 40960 data: 9 3 3 3 3 3 3 4 4 4 29/30 Read block 161 len 36864 30/30 Read block 512 len 16384 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 1 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 10 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Read 6 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:24.543682019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1430} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 59 59 59 114 114 114 T T T 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 60 60 60 115 115 115 F F F 4 080-099 62 62 62 121 121 121 F F F 5 100-119 60 60 60 117 117 117 F F F 6 120-139 61 61 61 119 119 119 T T T 7 140-159 58 58 58 113 113 113 F F F 8 160-179 60 60 60 118 118 118 F F F 9 180-199 60 60 60 118 118 118 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 62 62 62 121 121 121 F F F 13 260-279 62 62 62 121 121 121 T T T 14 280-299 60 60 60 116 116 116 F F F 15 300-319 62 62 62 121 121 121 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 57 57 57 112 112 112 T T T 18 360-379 62 62 62 121 121 121 T T T 19 380-399 62 62 62 120 120 120 T T T 20 400-419 60 60 60 116 116 116 F F F 21 420-439 59 59 59 114 114 114 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 59 59 59 114 114 114 F F F 25 500-519 59 59 59 114 114 114 T T T 26 520-539 61 61 61 119 119 119 F F F 27 540-559 59 59 59 114 114 114 T T T 28 560-579 61 61 61 119 119 119 T T T 29 580-599 56 56 56 111 111 111 F F F Max gen: 62, Max flush: 121 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 60 60 60 115 115 115 F F F 4 080-099 62 62 62 121 121 121 F F F 5 100-119 60 60 60 117 117 117 F F F 6 120-139 63 63 63 122 122 122 F F F 7 140-159 58 58 58 113 113 113 F F F 8 160-179 60 60 60 118 118 118 F F F 9 180-199 60 60 60 118 118 118 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 62 62 62 121 121 121 F F F 13 260-279 63 63 63 122 122 122 F F F 14 280-299 60 60 60 116 116 116 F F F 15 300-319 62 62 62 121 121 121 F F F 16 320-339 58 58 58 113 113 113 F F F 17 340-359 63 63 63 122 122 122 F F F 18 360-379 63 63 63 122 122 122 F F F 19 380-399 63 63 63 122 122 122 F F F 20 400-419 60 60 60 116 116 116 F F F 21 420-439 59 59 59 114 114 114 F F F 22 440-459 52 52 52 103 103 103 F F F 23 460-479 52 52 52 103 103 103 F F F 24 480-499 59 59 59 114 114 114 F F F 25 500-519 63 63 63 122 122 122 F F F 26 520-539 61 61 61 119 119 119 F F F 27 540-559 63 63 63 122 122 122 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 63, Max flush: 122 Begin loop 32 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.908559461Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:28.953416221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.953799318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.98505036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.98560875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.985647528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.985662697Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.985687957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} {"msg":"Crucible c0481e5a-af03-4ea5-a7a1-b9a3d6bfffb7 has session id: 8412de4c-ab36-4f88-8e95-645025a845df","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.98614328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Upstairs opts: Upstairs UUID: c0481e5a-af03-4ea5-a7a1-b9a3d6bfffb7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.986184709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Crucible stats registered with UUID: c0481e5a-af03-4ea5-a7a1-b9a3d6bfffb7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:28.986201968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015650595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015684294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} {"msg":"c0481e5a-af03-4ea5-a7a1-b9a3d6bfffb7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015743952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.01577675Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015802869Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015818619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015832879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015846578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015860407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015873777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015887637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015901516Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_connection connected from Ok(127.0.0.1:50011)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.015955034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"io task","client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_connection connected from Ok(127.0.0.1:61664)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.016246833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"io task","client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_connection connected from Ok(127.0.0.1:58308)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.016314751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"io task","client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.016653779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.016682518Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.016707707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.016736216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017240968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017272236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017289506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017304235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017881714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017907444Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.017921533Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.01799349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.01801462Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018028759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018041929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018540221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.01856446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[0]R flush_numbers[0..12]: [122, 114, 113, 115, 121, 117, 122, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018579879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[0]R generation[0..12]: [63, 59, 58, 60, 62, 60, 63, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018594289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018611778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[1]R flush_numbers[0..12]: [122, 114, 113, 115, 121, 117, 122, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018626658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[1]R generation[0..12]: [63, 59, 58, 60, 62, 60, 63, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018640737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018654957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[2]R flush_numbers[0..12]: [122, 114, 113, 115, 121, 117, 122, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018668706Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[2]R generation[0..12]: [63, 59, 58, 60, 62, 60, 63, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018682585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018696795Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018710015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018723334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Next flush: 123","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018736474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:05:29.018749653Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"":"downstairs","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018770053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018782252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"0","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018795752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"1","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018811131Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"client":"2","":"downstairs","session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018824931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.01883752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"c0481e5a-af03-4ea5-a7a1-b9a3d6bfffb7 is now active with session: 8412de4c-ab36-4f88-8e95-645025a845df","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.01885037Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018862429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438,"session_id":"8412de4c-ab36-4f88-8e95-645025a845df"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:29.018874959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 322 len 36864 data: 7 5 4 4 3 4 4 5 7 02/30 Read block 441 len 28672 03/30 Read block 112 len 8192 04/30 Write block 99 len 32768 data: 3 3 4 5 5 6 6 5 05/30 Read block 371 len 36864 06/30 Read block 59 len 8192 07/30 Read block 520 len 12288 08/30 Read block 146 len 20480 09/30 Read block 504 len 28672 10/30 Write block 446 len 32768 data: 3 4 3 3 3 2 3 6 11/30 Write block 307 len 4096 data: 6 12/30 Flush 13/30 Write block 473 len 28672 data: 6 6 6 4 3 2 3 14/30 Write block 265 len 12288 data: 5 6 8 15/30 Flush 16/30 Write block 74 len 16384 data: 4 5 6 4 17/30 Write block 445 len 36864 data: 3 4 5 4 4 4 3 4 7 18/30 Flush 19/30 Write block 128 len 8192 data: 8 6 20/30 Read block 556 len 36864 21/30 Write block 308 len 16384 data: 8 6 7 7 22/30 Flush 23/30 Write block 506 len 28672 data: 9 6 6 6 8 8 6 24/30 Write block 497 len 32768 data: 7 6 8 8 7 7 7 8 25/30 Read block 315 len 4096 26/30 Read block 349 len 16384 27/30 Write block 434 len 20480 data: 6 5 5 4 5 28/30 Read block 574 len 36864 29/30 Write block 548 len 40960 data: 10 11 10 9 10 10 4 4 4 4 30/30 Read block 332 len 32768 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:8 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1022 Acked Write 7 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 10 Sent Sent Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 0 4 Done 6 6 8 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1021 1021 1021 Downstairs last five completed: 1021 1020 1019 1018 1017 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:29.046364336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1438} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 64 64 64 126 126 126 F F F 7 140-159 58 58 58 113 113 113 F F F 8 160-179 60 60 60 118 118 118 F F F 9 180-199 60 60 60 118 118 118 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 62 62 62 121 121 121 F F F 13 260-279 64 64 64 124 124 124 F F F 14 280-299 60 60 60 116 116 116 F F F 15 300-319 64 64 64 126 126 126 F F F 16 320-339 64 64 64 123 123 123 F F F 17 340-359 63 63 63 122 122 122 F F F 18 360-379 63 63 63 122 122 122 F F F 19 380-399 63 63 63 122 122 122 F F F 20 400-419 60 60 60 116 116 116 F F F 21 420-439 59 59 59 114 114 114 T T T 22 440-459 64 64 64 125 125 125 F F F 23 460-479 64 64 64 124 124 124 F F F 24 480-499 59 59 59 114 114 114 T T T 25 500-519 63 63 63 122 122 122 T T T 26 520-539 61 61 61 119 119 119 F F F 27 540-559 63 63 63 122 122 122 T T T 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 64, Max flush: 126 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 64 64 64 126 126 126 F F F 7 140-159 58 58 58 113 113 113 F F F 8 160-179 60 60 60 118 118 118 F F F 9 180-199 60 60 60 118 118 118 F F F 10 200-219 59 59 59 114 114 114 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 62 62 62 121 121 121 F F F 13 260-279 64 64 64 124 124 124 F F F 14 280-299 60 60 60 116 116 116 F F F 15 300-319 64 64 64 126 126 126 F F F 16 320-339 64 64 64 123 123 123 F F F 17 340-359 63 63 63 122 122 122 F F F 18 360-379 63 63 63 122 122 122 F F F 19 380-399 63 63 63 122 122 122 F F F 20 400-419 60 60 60 116 116 116 F F F 21 420-439 65 65 65 127 127 127 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 64 64 64 124 124 124 F F F 24 480-499 65 65 65 127 127 127 F F F 25 500-519 65 65 65 127 127 127 F F F 26 520-539 61 61 61 119 119 119 F F F 27 540-559 65 65 65 127 127 127 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 65, Max flush: 127 Begin loop 33 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.374633401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.418018705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.418606814Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.418643293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.418658102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.418672372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} {"msg":"Crucible bd7e5c12-0c94-4153-90d6-14b9db0138a7 has session id: b23d905c-5554-416a-bfff-efac895348d3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.419175033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Upstairs opts: Upstairs UUID: bd7e5c12-0c94-4153-90d6-14b9db0138a7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.419215872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Crucible stats registered with UUID: bd7e5c12-0c94-4153-90d6-14b9db0138a7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.419232761Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.448962548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.448998167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} {"msg":"bd7e5c12-0c94-4153-90d6-14b9db0138a7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449063005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449098743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449128922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449160181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.44920092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449235078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449267997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449309526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449348054Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449532568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:34961)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449579926Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"io task","client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47937)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.449613885Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"io task","client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_connection connected from Ok(127.0.0.1:38647)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.44973145Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"io task","client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450051969Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450078448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450247942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450283481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.4503041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450318729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450660397Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.45083751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.450931787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451260865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451291984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451307133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451380951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.45140125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.45141555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451492717Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451512296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[0]R flush_numbers[0..12]: [122, 114, 113, 125, 123, 123, 126, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451526936Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[0]R generation[0..12]: [63, 59, 58, 64, 64, 64, 64, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451541035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451555324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[1]R flush_numbers[0..12]: [122, 114, 113, 125, 123, 123, 126, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451569394Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[1]R generation[0..12]: [63, 59, 58, 64, 64, 64, 64, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451583034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451600063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[2]R flush_numbers[0..12]: [122, 114, 113, 125, 123, 123, 126, 113, 118, 118, 114, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451614473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[2]R generation[0..12]: [63, 59, 58, 64, 64, 64, 64, 58, 60, 60, 59, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451628252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451642512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451655691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451669011Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Next flush: 128","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.45168294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451696379Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"":"downstairs","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451717359Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451729678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"0","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451743158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"1","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451756477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"client":"2","":"downstairs","session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451769887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451782116Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"bd7e5c12-0c94-4153-90d6-14b9db0138a7 is now active with session: b23d905c-5554-416a-bfff-efac895348d3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451794906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451809015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446,"session_id":"b23d905c-5554-416a-bfff-efac895348d3"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:33.451821575Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 249 len 16384 data: 5 5 9 9 02/30 Read block 509 len 16384 03/30 Read block 337 len 36864 04/30 Write block 317 len 28672 data: 9 9 9 8 7 8 6 05/30 Read block 462 len 36864 06/30 Write block 240 len 28672 data: 7 5 5 4 5 6 5 07/30 Read block 482 len 36864 08/30 Write block 305 len 4096 data: 6 09/30 Write block 407 len 20480 data: 4 5 7 7 6 10/30 Write block 204 len 40960 data: 8 9 9 8 6 9 7 6 4 6 11/30 Flush 12/30 Flush 13/30 Write block 158 len 40960 data: 5 6 7 5 7 5 6 7 5 5 14/30 Read block 510 len 28672 15/30 Write block 188 len 4096 data: 5 16/30 Write block 244 len 24576 data: 6 7 6 5 6 6 17/30 Write block 126 len 40960 data: 9 8 9 7 7 7 7 7 9 8 18/30 Write block 123 len 32768 data: 6 6 8 10 9 10 8 8 19/30 Read block 445 len 24576 20/30 Flush 21/30 Write block 510 len 32768 data: 9 9 7 4 5 4 5 7 22/30 Read block 428 len 4096 23/30 Read block 549 len 28672 24/30 Read block 124 len 36864 25/30 Write block 249 len 32768 data: 7 6 10 10 9 8 10 10 26/30 Read block 516 len 24576 27/30 Write block 354 len 36864 data: 5 4 4 4 5 6 8 7 9 28/30 Read block 277 len 40960 29/30 Read block 166 len 40960 30/30 Write block 287 len 20480 data: 9 8 6 8 9 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 8 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:33.479541384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1446} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 66 66 66 130 130 130 F F F 7 140-159 66 66 66 130 130 130 F F F 8 160-179 66 66 66 130 130 130 F F F 9 180-199 66 66 66 130 130 130 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 66 66 66 130 130 130 T T T 13 260-279 64 64 64 124 124 124 F F F 14 280-299 60 60 60 116 116 116 T T T 15 300-319 66 66 66 128 128 128 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 63 63 63 122 122 122 T T T 18 360-379 63 63 63 122 122 122 T T T 19 380-399 63 63 63 122 122 122 F F F 20 400-419 66 66 66 128 128 128 F F F 21 420-439 65 65 65 127 127 127 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 64 64 64 124 124 124 F F F 24 480-499 65 65 65 127 127 127 F F F 25 500-519 65 65 65 127 127 127 T T T 26 520-539 61 61 61 119 119 119 F F F 27 540-559 65 65 65 127 127 127 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 66, Max flush: 130 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 59 59 59 114 114 114 F F F 2 040-059 58 58 58 113 113 113 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 66 66 66 130 130 130 F F F 7 140-159 66 66 66 130 130 130 F F F 8 160-179 66 66 66 130 130 130 F F F 9 180-199 66 66 66 130 130 130 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 67 67 67 131 131 131 F F F 13 260-279 64 64 64 124 124 124 F F F 14 280-299 67 67 67 131 131 131 F F F 15 300-319 66 66 66 128 128 128 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 67 67 67 131 131 131 F F F 18 360-379 67 67 67 131 131 131 F F F 19 380-399 63 63 63 122 122 122 F F F 20 400-419 66 66 66 128 128 128 F F F 21 420-439 65 65 65 127 127 127 F F F 22 440-459 64 64 64 125 125 125 F F F 23 460-479 64 64 64 124 124 124 F F F 24 480-499 65 65 65 127 127 127 F F F 25 500-519 67 67 67 131 131 131 F F F 26 520-539 61 61 61 119 119 119 F F F 27 540-559 65 65 65 127 127 127 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 67, Max flush: 131 Begin loop 34 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.788541085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.831870171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.832501718Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.832549576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.832571816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.832590415Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} {"msg":"Crucible c23a5040-d775-4518-b9e9-c177581d58e7 has session id: 4e65faf5-41e3-47ea-a7e0-05a0da1321e2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.833081967Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Upstairs opts: Upstairs UUID: c23a5040-d775-4518-b9e9-c177581d58e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.833123176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Crucible stats registered with UUID: c23a5040-d775-4518-b9e9-c177581d58e7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.833146905Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.862908101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.862948049Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} {"msg":"c23a5040-d775-4518-b9e9-c177581d58e7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.862978188Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863005887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863054535Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863092774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863129203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863166211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.86320185Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863241329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863278417Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863312796Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:37379)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863398813Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"io task","client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:38761)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863435582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"io task","client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:39782)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.86347345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"io task","client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.863958383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864019231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864067679Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864097528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864124807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864178975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864542622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.86458504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.864618729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865169529Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865197808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865214048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865227527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865241476Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865255016Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865267466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865281005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[0]R flush_numbers[0..12]: [122, 114, 113, 125, 123, 123, 130, 130, 130, 130, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865295125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[0]R generation[0..12]: [63, 59, 58, 64, 64, 64, 66, 66, 66, 66, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865309154Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865323244Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[1]R flush_numbers[0..12]: [122, 114, 113, 125, 123, 123, 130, 130, 130, 130, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865337103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[1]R generation[0..12]: [63, 59, 58, 64, 64, 64, 66, 66, 66, 66, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865350692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865364812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[2]R flush_numbers[0..12]: [122, 114, 113, 125, 123, 123, 130, 130, 130, 130, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865378531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[2]R generation[0..12]: [63, 59, 58, 64, 64, 64, 66, 66, 66, 66, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865395361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.86540987Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865432569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865446049Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865459339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865472228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"":"downstairs","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865485428Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865497487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"0","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865510757Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"1","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865524186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"client":"2","":"downstairs","session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865537546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865549825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"c23a5040-d775-4518-b9e9-c177581d58e7 is now active with session: 4e65faf5-41e3-47ea-a7e0-05a0da1321e2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865562815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865575034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454,"session_id":"4e65faf5-41e3-47ea-a7e0-05a0da1321e2"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:37.865587534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 129 len 16384 data: 9 9 8 8 02/30 Write block 302 len 24576 data: 5 6 4 7 6 7 03/30 Write block 544 len 8192 data: 5 6 04/30 Read block 404 len 32768 05/30 Write block 548 len 32768 data: 11 12 11 10 11 11 5 5 06/30 Write block 29 len 40960 data: 5 5 6 6 6 8 6 6 8 8 07/30 Read block 170 len 4096 08/30 Flush 09/30 Write block 452 len 12288 data: 5 8 6 10/30 Write block 244 len 40960 data: 7 8 7 6 7 8 7 11 11 10 11/30 Read block 535 len 36864 12/30 Flush 13/30 Write block 369 len 36864 data: 9 11 11 11 10 9 9 7 7 14/30 Write block 50 len 4096 data: 5 15/30 Read block 267 len 24576 16/30 Write block 46 len 20480 data: 4 6 5 5 6 17/30 Read block 484 len 32768 18/30 Write block 154 len 20480 data: 3 5 6 6 6 19/30 Read block 204 len 28672 20/30 Write block 270 len 8192 data: 8 8 21/30 Flush 22/30 Write block 453 len 28672 data: 9 7 7 6 6 8 8 23/30 Write block 245 len 36864 data: 9 8 7 8 9 8 12 12 11 24/30 Flush 25/30 Write block 420 len 36864 data: 5 5 4 8 5 6 5 5 4 26/30 Read block 152 len 16384 27/30 Write block 395 len 24576 data: 9 8 8 9 10 9 28/30 Write block 512 len 24576 data: 8 5 6 5 6 8 29/30 Read block 449 len 12288 30/30 Write block 196 len 4096 data: 4 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 3 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 5 4 3 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:37.891395312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1454} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 68 68 68 132 132 132 F F F 2 040-059 68 68 68 134 134 134 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 68 68 68 134 134 134 F F F 8 160-179 66 66 66 130 130 130 F F F 9 180-199 66 66 66 130 130 130 T T T 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 68 68 68 135 135 135 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 67 67 67 131 131 131 F F F 15 300-319 68 68 68 132 132 132 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 67 67 67 131 131 131 F F F 18 360-379 68 68 68 134 134 134 F F F 19 380-399 63 63 63 122 122 122 T T T 20 400-419 66 66 66 128 128 128 T T T 21 420-439 65 65 65 127 127 127 T T T 22 440-459 68 68 68 135 135 135 F F F 23 460-479 64 64 64 124 124 124 F F F 24 480-499 65 65 65 127 127 127 F F F 25 500-519 67 67 67 131 131 131 T T T 26 520-539 61 61 61 119 119 119 F F F 27 540-559 68 68 68 132 132 132 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 68, Max flush: 135 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 F F F 1 020-039 68 68 68 132 132 132 F F F 2 040-059 68 68 68 134 134 134 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 68 68 68 132 132 132 F F F 7 140-159 68 68 68 134 134 134 F F F 8 160-179 66 66 66 130 130 130 F F F 9 180-199 69 69 69 136 136 136 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 68 68 68 135 135 135 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 67 67 67 131 131 131 F F F 15 300-319 68 68 68 132 132 132 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 67 67 67 131 131 131 F F F 18 360-379 68 68 68 134 134 134 F F F 19 380-399 69 69 69 136 136 136 F F F 20 400-419 69 69 69 136 136 136 F F F 21 420-439 69 69 69 136 136 136 F F F 22 440-459 68 68 68 135 135 135 F F F 23 460-479 64 64 64 124 124 124 F F F 24 480-499 65 65 65 127 127 127 F F F 25 500-519 69 69 69 136 136 136 F F F 26 520-539 61 61 61 119 119 119 F F F 27 540-559 68 68 68 132 132 132 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 56 56 56 111 111 111 F F F Max gen: 69, Max flush: 136 Begin loop 35 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.201758344Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.244786771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.245387979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.245423448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.245437477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.245451867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} {"msg":"Crucible 60ac07f0-bdcb-45fc-b236-d5937e7a8e42 has session id: 063d23ef-d70c-4910-ac2b-b38f5dbaa6fb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.245939429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Upstairs opts: Upstairs UUID: 60ac07f0-bdcb-45fc-b236-d5937e7a8e42, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.245992017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Crucible stats registered with UUID: 60ac07f0-bdcb-45fc-b236-d5937e7a8e42","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.246023746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27609219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276126099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} {"msg":"60ac07f0-bdcb-45fc-b236-d5937e7a8e42 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276197977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276220846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276247985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276265714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276282994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276299893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276319892Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276352561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27638729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276421019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:56775)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276463487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"io task","client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43990)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.276499026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"io task","client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33652)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27665048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"io task","client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277025657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277052636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27721606Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277251039Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277374744Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277406473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277688393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.277822598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278102618Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278290101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27832897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278354809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278380588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278406317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278430716Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278681977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278720216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[0]R flush_numbers[0..12]: [122, 132, 134, 125, 123, 123, 132, 134, 130, 136, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278747465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[0]R generation[0..12]: [63, 68, 68, 64, 64, 64, 68, 68, 66, 69, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278773454Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278800683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[1]R flush_numbers[0..12]: [122, 132, 134, 125, 123, 123, 132, 134, 130, 136, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278824992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[1]R generation[0..12]: [63, 68, 68, 64, 64, 64, 68, 68, 66, 69, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278850181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27887811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[2]R flush_numbers[0..12]: [122, 132, 134, 125, 123, 123, 132, 134, 130, 136, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278906069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[2]R generation[0..12]: [63, 68, 68, 64, 64, 64, 68, 68, 66, 69, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278931828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:05:42.278962107Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.278996596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279028304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Next flush: 137","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279057163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279085942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"":"downstairs","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279115101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.27914094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"0","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279169709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"1","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279201598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"client":"2","":"downstairs","session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279230687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279254356Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"60ac07f0-bdcb-45fc-b236-d5937e7a8e42 is now active with session: 063d23ef-d70c-4910-ac2b-b38f5dbaa6fb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279268336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279280855Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462,"session_id":"063d23ef-d70c-4910-ac2b-b38f5dbaa6fb"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:42.279293245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 588 len 8192 data: 11 10 02/30 Flush 03/30 Write block 343 len 40960 data: 6 6 5 5 5 6 6 3 3 3 04/30 Read block 474 len 24576 05/30 Read block 149 len 36864 06/30 Read block 406 len 36864 07/30 Write block 496 len 8192 data: 8 8 08/30 Read block 231 len 8192 09/30 Read block 442 len 28672 10/30 Write block 437 len 4096 data: 5 11/30 Write block 177 len 36864 data: 5 4 5 6 6 6 6 5 6 12/30 Read block 485 len 8192 13/30 Write block 503 len 4096 data: 8 14/30 Read block 97 len 16384 15/30 Read block 444 len 32768 16/30 Read block 428 len 28672 17/30 Write block 180 len 8192 data: 7 7 18/30 Read block 417 len 4096 19/30 Write block 536 len 12288 data: 4 4 5 20/30 Read block 124 len 20480 21/30 Read block 73 len 28672 22/30 Read block 84 len 16384 23/30 Read block 268 len 20480 24/30 Write block 10 len 36864 data: 3 3 3 7 7 6 6 8 8 25/30 Write block 145 len 28672 data: 5 5 5 4 2 3 3 26/30 Write block 465 len 12288 data: 4 5 5 27/30 Write block 127 len 8192 data: 10 11 28/30 Read block 296 len 24576 29/30 Write block 546 len 36864 data: 9 9 12 13 12 11 12 12 6 30/30 Read block 77 len 12288 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 9 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 2 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 9 Done Done Done false 1011 Acked Read 2 Done Done Done false 1012 Acked Write 1 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 5 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 3 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 9 Done Done Done false 1029 Acked Read 3 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 28 27 28 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:42.298397655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1462} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 63 63 63 122 122 122 T T T 1 020-039 68 68 68 132 132 132 F F F 2 040-059 68 68 68 134 134 134 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 68 68 68 132 132 132 T T T 7 140-159 68 68 68 134 134 134 T T T 8 160-179 66 66 66 130 130 130 T T T 9 180-199 69 69 69 136 136 136 T T T 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 68 68 68 135 135 135 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 67 67 67 131 131 131 F F F 15 300-319 68 68 68 132 132 132 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 67 67 67 131 131 131 T T T 18 360-379 68 68 68 134 134 134 F F F 19 380-399 69 69 69 136 136 136 F F F 20 400-419 69 69 69 136 136 136 F F F 21 420-439 69 69 69 136 136 136 T T T 22 440-459 68 68 68 135 135 135 F F F 23 460-479 64 64 64 124 124 124 T T T 24 480-499 65 65 65 127 127 127 T T T 25 500-519 69 69 69 136 136 136 T T T 26 520-539 61 61 61 119 119 119 T T T 27 540-559 68 68 68 132 132 132 T T T 28 560-579 63 63 63 122 122 122 F F F 29 580-599 70 70 70 137 137 137 F F F Max gen: 70, Max flush: 137 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 138 138 138 F F F 1 020-039 68 68 68 132 132 132 F F F 2 040-059 68 68 68 134 134 134 F F F 3 060-079 64 64 64 125 125 125 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 71 71 71 138 138 138 F F F 7 140-159 71 71 71 138 138 138 F F F 8 160-179 71 71 71 138 138 138 F F F 9 180-199 71 71 71 138 138 138 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 68 68 68 135 135 135 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 67 67 67 131 131 131 F F F 15 300-319 68 68 68 132 132 132 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 71 71 71 138 138 138 F F F 18 360-379 68 68 68 134 134 134 F F F 19 380-399 69 69 69 136 136 136 F F F 20 400-419 69 69 69 136 136 136 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 68 68 68 135 135 135 F F F 23 460-479 71 71 71 138 138 138 F F F 24 480-499 71 71 71 138 138 138 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 70 70 70 137 137 137 F F F Max gen: 71, Max flush: 138 Begin loop 36 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.699112156Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:46.741061322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.741453948Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.772280015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.772811405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.772854464Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.772875413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.772894103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} {"msg":"Crucible c7e9a393-1dfb-44fc-9c82-718fc10f2c83 has session id: 61c55707-87c5-4a15-8307-289ad677f51f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.773293818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Upstairs opts: Upstairs UUID: c7e9a393-1dfb-44fc-9c82-718fc10f2c83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.773342846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Crucible stats registered with UUID: c7e9a393-1dfb-44fc-9c82-718fc10f2c83","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.773372295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.802982456Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803028505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} {"msg":"c7e9a393-1dfb-44fc-9c82-718fc10f2c83 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803060434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803091523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803138861Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803177129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803212898Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803249007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803285625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803320714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803358133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803392952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_connection connected from Ok(127.0.0.1:56855)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803425491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"io task","client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_connection connected from Ok(127.0.0.1:41917)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803459729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"io task","client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_connection connected from Ok(127.0.0.1:49088)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.803549516Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"io task","client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804137585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804167874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804187523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804215862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804245441Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.80426546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804694455Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.804788291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.80482707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805167637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805210476Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805236965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805323732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805344951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805359271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.80538565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805399979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[0]R flush_numbers[0..12]: [138, 132, 134, 125, 123, 123, 138, 138, 138, 138, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805439178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[0]R generation[0..12]: [71, 68, 68, 64, 64, 64, 71, 71, 71, 71, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805457097Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805471887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[1]R flush_numbers[0..12]: [138, 132, 134, 125, 123, 123, 138, 138, 138, 138, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805486226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[1]R generation[0..12]: [71, 68, 68, 64, 64, 64, 71, 71, 71, 71, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805499996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805514315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[2]R flush_numbers[0..12]: [138, 132, 134, 125, 123, 123, 138, 138, 138, 138, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805528715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[2]R generation[0..12]: [71, 68, 68, 64, 64, 64, 71, 71, 71, 71, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805545084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805559523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805573093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805586633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805599952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805613132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"":"downstairs","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:05:46.805626571Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.8056457Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"0","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.80566021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"1","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805673589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"client":"2","":"downstairs","session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805686929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805699308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"c7e9a393-1dfb-44fc-9c82-718fc10f2c83 is now active with session: 61c55707-87c5-4a15-8307-289ad677f51f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805712198Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805724417Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470,"session_id":"61c55707-87c5-4a15-8307-289ad677f51f"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:46.805736807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 62 len 40960 data: 9 8 6 6 6 5 4 5 6 6 02/30 Write block 355 len 20480 data: 5 5 5 6 7 03/30 Flush 04/30 Write block 298 len 28672 data: 5 5 4 6 6 7 5 05/30 Read block 364 len 40960 06/30 Read block 321 len 24576 07/30 Read block 82 len 40960 08/30 Write block 137 len 40960 data: 7 4 4 3 3 3 4 4 6 6 09/30 Read block 337 len 20480 10/30 Write block 397 len 16384 data: 9 10 11 10 11/30 Read block 549 len 32768 12/30 Read block 546 len 16384 13/30 Flush 14/30 Flush 15/30 Write block 139 len 4096 data: 5 16/30 Write block 449 len 8192 data: 5 5 17/30 Read block 518 len 40960 18/30 Read block 462 len 36864 19/30 Read block 33 len 24576 20/30 Read block 278 len 36864 21/30 Read block 350 len 32768 22/30 Write block 348 len 24576 data: 7 7 4 4 4 4 23/30 Write block 282 len 32768 data: 11 13 9 9 10 10 9 7 24/30 Write block 157 len 28672 data: 7 7 7 8 6 8 6 25/30 Read block 378 len 32768 26/30 Read block 126 len 20480 27/30 Read block 369 len 36864 28/30 Write block 459 len 12288 data: 9 6 8 29/30 Read block 451 len 32768 30/30 Read block 23 len 24576 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 1 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 9 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 16 15 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:46.830883959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1470} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 138 138 138 F F F 1 020-039 68 68 68 132 132 132 F F F 2 040-059 68 68 68 134 134 134 F F F 3 060-079 72 72 72 139 139 139 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 72 72 72 140 140 140 T T T 7 140-159 72 72 72 140 140 140 T T T 8 160-179 71 71 71 138 138 138 T T T 9 180-199 71 71 71 138 138 138 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 68 68 68 135 135 135 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 72 72 72 140 140 140 T T T 15 300-319 72 72 72 140 140 140 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 72 72 72 139 139 139 T T T 18 360-379 68 68 68 134 134 134 F F F 19 380-399 72 72 72 140 140 140 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 68 68 68 135 135 135 T T T 23 460-479 71 71 71 138 138 138 T T T 24 480-499 71 71 71 138 138 138 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 70 70 70 137 137 137 F F F Max gen: 72, Max flush: 140 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 138 138 138 F F F 1 020-039 68 68 68 132 132 132 F F F 2 040-059 68 68 68 134 134 134 F F F 3 060-079 72 72 72 139 139 139 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 F F F 6 120-139 73 73 73 141 141 141 F F F 7 140-159 73 73 73 141 141 141 F F F 8 160-179 73 73 73 141 141 141 F F F 9 180-199 71 71 71 138 138 138 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 58 58 58 113 113 113 F F F 12 240-259 68 68 68 135 135 135 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 73 73 73 141 141 141 F F F 15 300-319 72 72 72 140 140 140 F F F 16 320-339 66 66 66 128 128 128 F F F 17 340-359 73 73 73 141 141 141 F F F 18 360-379 68 68 68 134 134 134 F F F 19 380-399 72 72 72 140 140 140 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 73 73 73 141 141 141 F F F 23 460-479 73 73 73 141 141 141 F F F 24 480-499 71 71 71 138 138 138 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 63 63 63 122 122 122 F F F 29 580-599 70 70 70 137 137 137 F F F Max gen: 73, Max flush: 141 Begin loop 37 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.198438515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:51.241501061Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.241887216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.27338805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.27392259Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.273964319Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.273987078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.274005837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} {"msg":"Crucible 265595b4-94c4-4e19-937e-6dbc23d69bc7 has session id: 9391e4eb-f33e-43ba-82ff-8965176a0033","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.274518999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Upstairs opts: Upstairs UUID: 265595b4-94c4-4e19-937e-6dbc23d69bc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.274562687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Crucible stats registered with UUID: 265595b4-94c4-4e19-937e-6dbc23d69bc7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.274588296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304471127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304499976Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} {"msg":"265595b4-94c4-4e19-937e-6dbc23d69bc7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304577893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304601733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304630892Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304648941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.30467263Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304704459Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304736668Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304770087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304837954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.304887672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_connection connected from Ok(127.0.0.1:62475)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305094055Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"io task","client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_connection connected from Ok(127.0.0.1:63782)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305129314Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"io task","client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_connection connected from Ok(127.0.0.1:40806)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305160363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"io task","client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305678394Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305706863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305722482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305736572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305756611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.305770561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.306408418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.306449836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.306583921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307053944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307086293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307103422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.30716413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307196929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307213089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307226128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307239828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[0]R flush_numbers[0..12]: [138, 132, 134, 139, 123, 123, 141, 141, 141, 138, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307258267Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[0]R generation[0..12]: [71, 68, 68, 72, 64, 64, 73, 73, 73, 71, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307272246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307286416Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[1]R flush_numbers[0..12]: [138, 132, 134, 139, 123, 123, 141, 141, 141, 138, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307300305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[1]R generation[0..12]: [71, 68, 68, 72, 64, 64, 73, 73, 73, 71, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307313945Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307328114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[2]R flush_numbers[0..12]: [138, 132, 134, 139, 123, 123, 141, 141, 141, 138, 128, 113]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307342044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[2]R generation[0..12]: [71, 68, 68, 72, 64, 64, 73, 73, 73, 71, 66, 58]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307359323Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307373863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Max found gen is 74","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307387102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307400342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Next flush: 142","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307413411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307426421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307502008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307547956Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"0","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307564226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"1","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307578435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"client":"2","":"downstairs","session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307592405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307605084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"265595b4-94c4-4e19-937e-6dbc23d69bc7 is now active with session: 9391e4eb-f33e-43ba-82ff-8965176a0033","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307617964Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307630203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478,"session_id":"9391e4eb-f33e-43ba-82ff-8965176a0033"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:51.307642733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 281 len 36864 data: 10 12 14 10 10 11 11 10 8 02/30 Write block 296 len 20480 data: 4 4 6 6 5 03/30 Read block 321 len 4096 04/30 Write block 38 len 36864 data: 9 8 7 7 6 6 5 5 5 05/30 Flush 06/30 Write block 43 len 16384 data: 7 6 6 6 07/30 Write block 137 len 16384 data: 8 5 6 4 08/30 Read block 464 len 28672 09/30 Read block 153 len 28672 10/30 Read block 164 len 12288 11/30 Write block 457 len 24576 data: 7 9 10 7 9 7 12/30 Read block 383 len 28672 13/30 Read block 28 len 16384 14/30 Write block 473 len 20480 data: 7 7 7 5 4 15/30 Write block 233 len 32768 data: 5 3 2 3 4 5 7 8 16/30 Write block 231 len 28672 data: 6 5 6 4 3 4 5 17/30 Write block 283 len 28672 data: 15 11 11 12 12 11 9 18/30 Flush 19/30 Write block 459 len 36864 data: 11 8 10 8 7 5 5 6 6 20/30 Read block 121 len 36864 21/30 Read block 521 len 16384 22/30 Write block 317 len 20480 data: 10 10 10 9 8 23/30 Write block 242 len 12288 data: 6 5 8 24/30 Write block 589 len 8192 data: 11 9 25/30 Write block 16 len 8192 data: 7 9 26/30 Write block 112 len 32768 data: 6 7 7 8 8 8 7 9 27/30 Write block 479 len 20480 data: 4 4 4 5 5 28/30 Write block 567 len 4096 data: 10 29/30 Read block 269 len 36864 30/30 Read block 219 len 28672 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:12 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1018 Acked Write 9 Done Done Done false 1019 Acked Read 9 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Write 8 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 11 12 12 35 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1017 1017 1017 Downstairs last five completed: 1017 1016 1015 1014 1013 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:51.330456269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1478} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 71 71 71 138 138 138 T T T 1 020-039 74 74 74 142 142 142 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 72 72 72 139 139 139 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 64 64 64 123 123 123 T T T 6 120-139 74 74 74 143 143 143 F F F 7 140-159 74 74 74 143 143 143 F F F 8 160-179 73 73 73 141 141 141 F F F 9 180-199 71 71 71 138 138 138 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 74 74 74 143 143 143 F F F 12 240-259 74 74 74 143 143 143 T T T 13 260-279 68 68 68 134 134 134 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 74 74 74 142 142 142 T T T 16 320-339 66 66 66 128 128 128 T T T 17 340-359 73 73 73 141 141 141 F F F 18 360-379 68 68 68 134 134 134 F F F 19 380-399 72 72 72 140 140 140 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 74 74 74 143 143 143 T T T 23 460-479 74 74 74 143 143 143 T T T 24 480-499 71 71 71 138 138 138 T T T 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 63 63 63 122 122 122 T T T 29 580-599 70 70 70 137 137 137 T T T Max gen: 74, Max flush: 143 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 144 144 144 F F F 1 020-039 74 74 74 142 142 142 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 72 72 72 139 139 139 F F F 4 080-099 64 64 64 123 123 123 F F F 5 100-119 75 75 75 144 144 144 F F F 6 120-139 74 74 74 143 143 143 F F F 7 140-159 74 74 74 143 143 143 F F F 8 160-179 73 73 73 141 141 141 F F F 9 180-199 71 71 71 138 138 138 F F F 10 200-219 66 66 66 128 128 128 F F F 11 220-239 74 74 74 143 143 143 F F F 12 240-259 75 75 75 144 144 144 F F F 13 260-279 68 68 68 134 134 134 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 75 75 75 144 144 144 F F F 17 340-359 73 73 73 141 141 141 F F F 18 360-379 68 68 68 134 134 134 F F F 19 380-399 72 72 72 140 140 140 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 75 75 75 144 144 144 F F F 23 460-479 75 75 75 144 144 144 F F F 24 480-499 75 75 75 144 144 144 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 75, Max flush: 144 Begin loop 38 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.739851825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.783433311Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.784320589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.784359438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.784374217Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.784388627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} {"msg":"Crucible 9e6bd14c-f79c-4b52-b945-0fc5f04f8e06 has session id: 5bf2d1ad-4369-467d-9691-9678e6bc8d98","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.784943167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Upstairs opts: Upstairs UUID: 9e6bd14c-f79c-4b52-b945-0fc5f04f8e06, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.784984555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Crucible stats registered with UUID: 9e6bd14c-f79c-4b52-b945-0fc5f04f8e06","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.785010024Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.814883846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.814912915Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} {"msg":"9e6bd14c-f79c-4b52-b945-0fc5f04f8e06 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.814984272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815011161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81504443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815060809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815075789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815096468Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815184885Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815221814Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815250282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815278681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_connection connected from Ok(127.0.0.1:35463)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81531195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"io task","client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_connection connected from Ok(127.0.0.1:36565)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815353189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"io task","client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_connection connected from Ok(127.0.0.1:47402)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.815402977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"io task","client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816086882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816115231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816131321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81614552Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816215578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816237617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816794157Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816837175Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.816872904Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817363516Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817394245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817410324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817478672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817502231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817516371Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81752891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81754259Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[0]R flush_numbers[0..12]: [144, 142, 143, 139, 123, 144, 143, 143, 141, 138, 128, 143]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817556829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[0]R generation[0..12]: [75, 74, 74, 72, 64, 75, 74, 74, 73, 71, 66, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817570859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817585178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[1]R flush_numbers[0..12]: [144, 142, 143, 139, 123, 144, 143, 143, 141, 138, 128, 143]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817599108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[1]R generation[0..12]: [75, 74, 74, 72, 64, 75, 74, 74, 73, 71, 66, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817612837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817627087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[2]R flush_numbers[0..12]: [144, 142, 143, 139, 123, 144, 143, 143, 141, 138, 128, 143]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817641136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[2]R generation[0..12]: [75, 74, 74, 72, 64, 75, 74, 74, 73, 71, 66, 74]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817654766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817668705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817684775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817698574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Next flush: 145","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817711674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817724813Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817747032Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817759282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"0","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817772502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"1","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817785901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"client":"2","":"downstairs","session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817799381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81781191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"9e6bd14c-f79c-4b52-b945-0fc5f04f8e06 is now active with session: 5bf2d1ad-4369-467d-9691-9678e6bc8d98","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.81782505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817837299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486,"session_id":"5bf2d1ad-4369-467d-9691-9678e6bc8d98"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:05:55.817849869Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 381 len 32768 data: 9 9 9 10 10 9 6 9 02/30 Read block 43 len 36864 03/30 Read block 136 len 16384 04/30 Read block 221 len 28672 05/30 Read block 531 len 12288 06/30 Write block 147 len 12288 data: 6 5 3 07/30 Read block 523 len 20480 08/30 Write block 461 len 36864 data: 11 9 8 6 6 7 7 5 5 09/30 Flush 10/30 Write block 136 len 20480 data: 8 9 6 7 5 11/30 Write block 271 len 12288 data: 9 9 7 12/30 Read block 485 len 36864 13/30 Read block 471 len 20480 14/30 Flush 15/30 Write block 149 len 20480 data: 4 4 4 4 3 16/30 Read block 429 len 4096 17/30 Read block 519 len 8192 18/30 Write block 355 len 12288 data: 6 6 6 19/30 Flush 20/30 Write block 177 len 28672 data: 6 5 6 8 8 7 7 21/30 Write block 378 len 16384 data: 6 8 9 10 22/30 Read block 375 len 36864 23/30 Write block 355 len 36864 data: 7 7 7 7 8 9 8 10 11 24/30 Read block 266 len 16384 25/30 Read block 335 len 16384 26/30 Write block 79 len 40960 data: 4 4 4 4 3 3 5 7 8 8 27/30 Write block 219 len 12288 data: 6 5 5 28/30 Read block 50 len 36864 29/30 Write block 28 len 12288 data: 7 6 6 30/30 Read block 47 len 12288 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 7 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 11 10 9 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:05:55.844018644Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1486} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 144 144 144 F F F 1 020-039 74 74 74 142 142 142 T T T 2 040-059 74 74 74 143 143 143 F F F 3 060-079 72 72 72 139 139 139 T T T 4 080-099 64 64 64 123 123 123 T T T 5 100-119 75 75 75 144 144 144 F F F 6 120-139 76 76 76 146 146 146 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 73 73 73 141 141 141 T T T 9 180-199 71 71 71 138 138 138 T T T 10 200-219 66 66 66 128 128 128 T T T 11 220-239 74 74 74 143 143 143 T T T 12 240-259 75 75 75 144 144 144 F F F 13 260-279 76 76 76 146 146 146 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 75 75 75 144 144 144 F F F 17 340-359 76 76 76 147 147 147 T T T 18 360-379 68 68 68 134 134 134 T T T 19 380-399 76 76 76 145 145 145 T T T 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 75 75 75 144 144 144 F F F 23 460-479 76 76 76 145 145 145 F F F 24 480-499 75 75 75 144 144 144 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 76, Max flush: 147 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 144 144 144 F F F 1 020-039 77 77 77 148 148 148 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 77 77 77 148 148 148 F F F 4 080-099 77 77 77 148 148 148 F F F 5 100-119 75 75 75 144 144 144 F F F 6 120-139 76 76 76 146 146 146 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 77 77 77 148 148 148 F F F 10 200-219 77 77 77 148 148 148 F F F 11 220-239 77 77 77 148 148 148 F F F 12 240-259 75 75 75 144 144 144 F F F 13 260-279 76 76 76 146 146 146 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 75 75 75 144 144 144 F F F 17 340-359 77 77 77 148 148 148 F F F 18 360-379 77 77 77 148 148 148 F F F 19 380-399 77 77 77 148 148 148 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 F F F 22 440-459 75 75 75 144 144 144 F F F 23 460-479 76 76 76 145 145 145 F F F 24 480-499 75 75 75 144 144 144 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 71 71 71 138 138 138 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 77, Max flush: 148 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.232759235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.276023893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.276592583Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.276643081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.27666363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.27668762Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} {"msg":"Crucible 77df0e73-1f39-4587-81f9-52ceb269df7f has session id: d62a6496-ddf3-43fb-ad10-b2f919b787bc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.277048846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Upstairs opts: Upstairs UUID: 77df0e73-1f39-4587-81f9-52ceb269df7f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.277092485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Crucible stats registered with UUID: 77df0e73-1f39-4587-81f9-52ceb269df7f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.277121754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.306978666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307019694Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} {"msg":"77df0e73-1f39-4587-81f9-52ceb269df7f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307059203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307078122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307106041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307127531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307160009Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307194978Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307232827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307267976Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307305504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307341303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:43098)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307372052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"io task","client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35745)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307400911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"io task","client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_connection connected from Ok(127.0.0.1:56478)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.307510297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"io task","client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308027538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308058307Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308074207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308088496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308237481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308283339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308734763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308776011Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.308909516Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309300092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309325171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309339971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309415028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309448527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309468426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309481296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309494795Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[0]R flush_numbers[0..12]: [144, 148, 143, 148, 148, 144, 146, 147, 148, 148, 148, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309508665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[0]R generation[0..12]: [75, 77, 74, 77, 77, 75, 76, 76, 77, 77, 77, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309522504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309536574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[1]R flush_numbers[0..12]: [144, 148, 143, 148, 148, 144, 146, 147, 148, 148, 148, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309550513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[1]R generation[0..12]: [75, 77, 74, 77, 77, 75, 76, 76, 77, 77, 77, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309564233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309579752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[2]R flush_numbers[0..12]: [144, 148, 143, 148, 148, 144, 146, 147, 148, 148, 148, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309593832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[2]R generation[0..12]: [75, 77, 74, 77, 77, 75, 76, 76, 77, 77, 77, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309607731Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.30963026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309665819Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309688448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Next flush: 149","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309705057Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309718767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"":"downstairs","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309732347Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309744436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"0","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309757756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"1","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309771035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"client":"2","":"downstairs","session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309784375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309797034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"77df0e73-1f39-4587-81f9-52ceb269df7f is now active with session: d62a6496-ddf3-43fb-ad10-b2f919b787bc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309810094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309822363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494,"session_id":"d62a6496-ddf3-43fb-ad10-b2f919b787bc"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:00.309834663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 127 len 4096 data: 11 02/30 Read block 562 len 24576 03/30 Write block 116 len 8192 data: 9 9 04/30 Read block 401 len 36864 05/30 Read block 366 len 8192 06/30 Read block 33 len 4096 07/30 Read block 67 len 36864 08/30 Write block 202 len 12288 data: 6 7 9 09/30 Write block 536 len 12288 data: 5 5 6 10/30 Write block 335 len 16384 data: 10 10 9 9 11/30 Read block 326 len 36864 12/30 Read block 472 len 16384 13/30 Read block 294 len 8192 14/30 Read block 442 len 12288 15/30 Read block 208 len 36864 16/30 Write block 25 len 12288 data: 7 6 6 17/30 Write block 395 len 16384 data: 10 9 10 11 18/30 Read block 214 len 40960 19/30 Read block 439 len 36864 20/30 Read block 208 len 20480 21/30 Read block 375 len 32768 22/30 Write block 368 len 32768 data: 7 10 12 12 12 11 10 10 23/30 Flush 24/30 Write block 421 len 28672 data: 6 5 9 6 7 6 6 25/30 Read block 64 len 24576 26/30 Write block 271 len 32768 data: 10 10 8 9 8 8 9 9 27/30 Write block 272 len 16384 data: 11 9 10 9 28/30 Read block 73 len 28672 29/30 Write block 88 len 40960 data: 9 8 10 7 8 9 10 9 7 6 30/30 Write block 331 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 7 Sent Done Sent false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 2 5 10 Done 4 5 2 11 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:00.331147843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1494} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 144 144 144 F F F 1 020-039 78 78 78 149 149 149 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 77 77 77 148 148 148 F F F 4 080-099 77 77 77 148 148 148 T T T 5 100-119 78 78 78 149 149 149 F F F 6 120-139 78 78 78 149 149 149 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 77 77 77 148 148 148 F F F 10 200-219 78 78 78 149 149 149 F F F 11 220-239 77 77 77 148 148 148 F F F 12 240-259 75 75 75 144 144 144 F F F 13 260-279 76 76 76 146 146 146 T T T 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 78 78 78 149 149 149 T T T 17 340-359 77 77 77 148 148 148 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 78 78 78 149 149 149 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 71 71 71 138 138 138 T T T 22 440-459 75 75 75 144 144 144 F F F 23 460-479 76 76 76 145 145 145 F F F 24 480-499 75 75 75 144 144 144 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 78 78 78 149 149 149 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 78, Max flush: 149 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 75 75 75 144 144 144 F F F 1 020-039 78 78 78 149 149 149 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 77 77 77 148 148 148 F F F 4 080-099 79 79 79 150 150 150 F F F 5 100-119 78 78 78 149 149 149 F F F 6 120-139 78 78 78 149 149 149 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 77 77 77 148 148 148 F F F 10 200-219 78 78 78 149 149 149 F F F 11 220-239 77 77 77 148 148 148 F F F 12 240-259 75 75 75 144 144 144 F F F 13 260-279 79 79 79 150 150 150 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 79 79 79 150 150 150 F F F 17 340-359 77 77 77 148 148 148 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 78 78 78 149 149 149 F F F 20 400-419 72 72 72 140 140 140 F F F 21 420-439 79 79 79 150 150 150 F F F 22 440-459 75 75 75 144 144 144 F F F 23 460-479 76 76 76 145 145 145 F F F 24 480-499 75 75 75 144 144 144 F F F 25 500-519 71 71 71 138 138 138 F F F 26 520-539 78 78 78 149 149 149 F F F 27 540-559 71 71 71 138 138 138 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 79, Max flush: 150 Begin loop 40 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.628711897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:04.670646723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.671044149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.701777709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.702363878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.702397817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.702411906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.702426825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} {"msg":"Crucible df8d5859-7358-4e0b-bbf3-91f9fee03546 has session id: ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.702957626Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Upstairs opts: Upstairs UUID: df8d5859-7358-4e0b-bbf3-91f9fee03546, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.703009015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Crucible stats registered with UUID: df8d5859-7358-4e0b-bbf3-91f9fee03546","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.703029114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.732839918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.732866977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} {"msg":"df8d5859-7358-4e0b-bbf3-91f9fee03546 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.732923395Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.732958474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.732988762Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733007082Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733024211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73305831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733074939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733092329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733109308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_connection connected from Ok(127.0.0.1:36314)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733185085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"io task","client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47723)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733227824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"io task","client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_connection connected from Ok(127.0.0.1:35801)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.733404797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"io task","client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734003756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734038904Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734055684Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734143601Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73416791Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734183569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73471564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734745469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.734761468Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735330708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735358907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735373676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735438334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735458573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735472233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735484802Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735501112Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[0]R flush_numbers[0..12]: [144, 149, 143, 148, 150, 149, 149, 147, 148, 148, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735521751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[0]R generation[0..12]: [75, 78, 74, 77, 79, 78, 78, 76, 77, 77, 78, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73553609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73555086Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[1]R flush_numbers[0..12]: [144, 149, 143, 148, 150, 149, 149, 147, 148, 148, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735565219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[1]R generation[0..12]: [75, 78, 74, 77, 79, 78, 78, 76, 77, 77, 78, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735579029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735593278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[2]R flush_numbers[0..12]: [144, 149, 143, 148, 150, 149, 149, 147, 148, 148, 149, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735607278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[2]R generation[0..12]: [75, 78, 74, 77, 79, 78, 78, 76, 77, 77, 78, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735621057Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735635027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735648226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735661596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735674515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735687505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735774152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735797831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"0","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73581305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"1","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.73582711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"client":"2","":"downstairs","session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735841029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735853789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"df8d5859-7358-4e0b-bbf3-91f9fee03546 is now active with session: ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735867089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735879288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502,"session_id":"ffb8d3e7-3f5f-42c5-bf78-6d40ea22555b"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:04.735891797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 87 len 40960 data: 9 10 9 11 8 9 10 11 10 8 03/30 Write block 3 len 16384 data: 3 4 4 5 04/30 Read block 539 len 32768 05/30 Read block 281 len 40960 06/30 Read block 265 len 16384 07/30 Read block 592 len 8192 08/30 Read block 377 len 16384 09/30 Flush 10/30 Write block 465 len 24576 data: 7 8 8 6 6 6 11/30 Write block 507 len 8192 data: 7 7 12/30 Read block 540 len 8192 13/30 Read block 351 len 24576 14/30 Write block 502 len 12288 data: 8 9 9 15/30 Read block 218 len 36864 16/30 Write block 416 len 20480 data: 5 4 5 3 6 17/30 Write block 547 len 40960 data: 10 13 14 13 12 13 13 7 6 5 18/30 Write block 257 len 16384 data: 7 6 4 4 19/30 Read block 591 len 4096 20/30 Read block 130 len 20480 21/30 Flush 22/30 Write block 473 len 36864 data: 8 8 8 6 5 3 5 5 5 23/30 Write block 134 len 12288 data: 10 9 9 24/30 Write block 112 len 36864 data: 7 8 8 9 10 10 8 10 9 25/30 Write block 484 len 36864 data: 7 7 7 7 7 5 5 6 7 26/30 Read block 484 len 32768 27/30 Write block 217 len 4096 data: 8 28/30 Read block 222 len 8192 29/30 Read block 65 len 32768 30/30 Write block 341 len 28672 data: 5 7 7 7 6 6 6 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 9 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:04.758052258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1502} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 78 78 78 149 149 149 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 77 77 77 148 148 148 F F F 4 080-099 80 80 80 152 152 152 F F F 5 100-119 78 78 78 149 149 149 T T T 6 120-139 78 78 78 149 149 149 T T T 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 77 77 77 148 148 148 F F F 10 200-219 78 78 78 149 149 149 T T T 11 220-239 77 77 77 148 148 148 F F F 12 240-259 80 80 80 153 153 153 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 79 79 79 150 150 150 F F F 17 340-359 77 77 77 148 148 148 T T T 18 360-379 78 78 78 149 149 149 F F F 19 380-399 78 78 78 149 149 149 F F F 20 400-419 80 80 80 153 153 153 F F F 21 420-439 80 80 80 153 153 153 F F F 22 440-459 75 75 75 144 144 144 F F F 23 460-479 80 80 80 153 153 153 T T T 24 480-499 75 75 75 144 144 144 T T T 25 500-519 80 80 80 153 153 153 F F F 26 520-539 78 78 78 149 149 149 F F F 27 540-559 80 80 80 153 153 153 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 80, Max flush: 153 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 78 78 78 149 149 149 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 77 77 77 148 148 148 F F F 4 080-099 80 80 80 152 152 152 F F F 5 100-119 81 81 81 154 154 154 F F F 6 120-139 81 81 81 154 154 154 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 77 77 77 148 148 148 F F F 10 200-219 81 81 81 154 154 154 F F F 11 220-239 77 77 77 148 148 148 F F F 12 240-259 80 80 80 153 153 153 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 74 74 74 143 143 143 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 79 79 79 150 150 150 F F F 17 340-359 81 81 81 154 154 154 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 78 78 78 149 149 149 F F F 20 400-419 80 80 80 153 153 153 F F F 21 420-439 80 80 80 153 153 153 F F F 22 440-459 75 75 75 144 144 144 F F F 23 460-479 81 81 81 154 154 154 F F F 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 78 78 78 149 149 149 F F F 27 540-559 80 80 80 153 153 153 F F F 28 560-579 75 75 75 144 144 144 F F F 29 580-599 75 75 75 144 144 144 F F F Max gen: 81, Max flush: 154 Begin loop 41 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.113031328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.156619374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.157247992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.15728553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.15730067Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.157315409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} {"msg":"Crucible d49d8cd7-ae43-4bfe-8052-d60a97f1379d has session id: f1909041-4698-4e9a-b24d-d74263ab393e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.157734764Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Upstairs opts: Upstairs UUID: d49d8cd7-ae43-4bfe-8052-d60a97f1379d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.157796692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Crucible stats registered with UUID: d49d8cd7-ae43-4bfe-8052-d60a97f1379d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.157823461Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.187798319Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.187835058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} {"msg":"d49d8cd7-ae43-4bfe-8052-d60a97f1379d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.187870317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.187892536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.187944154Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.187982713Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188019561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188098448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188135927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188165486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188192205Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188218174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_connection connected from Ok(127.0.0.1:59181)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188243393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"io task","client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34894)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188267162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"io task","client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_connection connected from Ok(127.0.0.1:42045)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188415707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"io task","client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.18887628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188914819Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188945968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.188977686Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.189048494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.189130101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.189563125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.189596724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.189627003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190156534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190195953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190212012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.19027295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190293329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190307538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190320118Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190333657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[0]R flush_numbers[0..12]: [152, 149, 143, 148, 152, 154, 154, 147, 148, 148, 154, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190347627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[0]R generation[0..12]: [80, 78, 74, 77, 80, 81, 81, 76, 77, 77, 81, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190361656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190378876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[1]R flush_numbers[0..12]: [152, 149, 143, 148, 152, 154, 154, 147, 148, 148, 154, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190393205Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[1]R generation[0..12]: [80, 78, 74, 77, 80, 81, 81, 76, 77, 77, 81, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190406985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190421084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[2]R flush_numbers[0..12]: [152, 149, 143, 148, 152, 154, 154, 147, 148, 148, 154, 148]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190435374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[2]R generation[0..12]: [80, 78, 74, 77, 80, 81, 81, 76, 77, 77, 81, 77]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190449283Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190463383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190476402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:06:09.190489522Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190511431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190524601Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"":"downstairs","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.19053793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.19054996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"0","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190563149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"1","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190576549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"client":"2","":"downstairs","session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190591818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190604418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"d49d8cd7-ae43-4bfe-8052-d60a97f1379d is now active with session: f1909041-4698-4e9a-b24d-d74263ab393e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190617367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190629477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510,"session_id":"f1909041-4698-4e9a-b24d-d74263ab393e"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:09.190641997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 189 len 4096 data: 7 02/30 Write block 79 len 32768 data: 5 5 5 5 4 4 6 8 03/30 Write block 24 len 32768 data: 6 8 7 7 8 7 7 7 04/30 Write block 243 len 32768 data: 6 9 10 9 8 9 10 9 05/30 Write block 283 len 16384 data: 16 12 12 13 06/30 Write block 100 len 28672 data: 4 5 6 6 7 7 6 07/30 Write block 25 len 12288 data: 9 8 8 08/30 Write block 395 len 32768 data: 11 10 11 12 12 11 7 5 09/30 Read block 38 len 32768 10/30 Read block 297 len 16384 11/30 Write block 419 len 36864 data: 4 7 7 6 10 7 8 7 7 12/30 Write block 251 len 8192 data: 13 13 13/30 Write block 457 len 4096 data: 8 14/30 Read block 451 len 32768 15/30 Read block 85 len 36864 16/30 Write block 531 len 32768 data: 7 7 6 6 5 6 6 7 17/30 Write block 332 len 20480 data: 9 10 10 11 11 18/30 Read block 101 len 16384 19/30 Write block 554 len 16384 data: 8 7 6 5 20/30 Write block 108 len 4096 data: 5 21/30 Read block 421 len 20480 22/30 Read block 449 len 20480 23/30 Write block 215 len 40960 data: 5 8 9 7 7 6 6 6 4 3 24/30 Flush 25/30 Write block 419 len 20480 data: 5 8 8 7 11 26/30 Read block 434 len 32768 27/30 Read block 513 len 4096 28/30 Write block 291 len 36864 data: 10 7 5 3 4 5 5 7 7 29/30 Write block 462 len 16384 data: 10 9 7 8 30/30 Write block 575 len 28672 data: 4 4 5 5 4 4 3 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 5 Done Done Done false 1025 Acked Read 8 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 5 9 Done 4 4 1 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:09.212750098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1510} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 82 82 82 155 155 155 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 82 82 82 155 155 155 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 82 82 82 155 155 155 F F F 6 120-139 81 81 81 154 154 154 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 82 82 82 155 155 155 F F F 10 200-219 82 82 82 155 155 155 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 82 82 82 155 155 155 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 82 82 82 155 155 155 T T T 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 81 81 81 154 154 154 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 82 82 82 155 155 155 F F F 20 400-419 82 82 82 155 155 155 T T T 21 420-439 82 82 82 155 155 155 T T T 22 440-459 82 82 82 155 155 155 F F F 23 460-479 81 81 81 154 154 154 T T T 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 75 75 75 144 144 144 T T T 29 580-599 75 75 75 144 144 144 T T T Max gen: 82, Max flush: 155 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 82 82 82 155 155 155 F F F 2 040-059 74 74 74 143 143 143 F F F 3 060-079 82 82 82 155 155 155 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 82 82 82 155 155 155 F F F 6 120-139 81 81 81 154 154 154 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 82 82 82 155 155 155 F F F 10 200-219 82 82 82 155 155 155 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 82 82 82 155 155 155 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 83 83 83 156 156 156 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 81 81 81 154 154 154 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 82 82 82 155 155 155 F F F 20 400-419 83 83 83 156 156 156 F F F 21 420-439 83 83 83 156 156 156 F F F 22 440-459 82 82 82 155 155 155 F F F 23 460-479 83 83 83 156 156 156 F F F 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 83, Max flush: 156 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.53420708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.577984839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.578607817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.578644445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.578659165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.578673594Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} {"msg":"Crucible 3cdcc472-c077-4f2f-b3da-29e0941515b4 has session id: 020c9af3-fab9-4340-836f-ffd96050c208","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.579236874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Upstairs opts: Upstairs UUID: 3cdcc472-c077-4f2f-b3da-29e0941515b4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.579290612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Crucible stats registered with UUID: 3cdcc472-c077-4f2f-b3da-29e0941515b4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.579320181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609050628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609097146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} {"msg":"3cdcc472-c077-4f2f-b3da-29e0941515b4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609166714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609199382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609230011Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609248561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.60926942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609300099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609331568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609361816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609395085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609429214Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_connection connected from Ok(127.0.0.1:65280)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609464953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"io task","client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_connection connected from Ok(127.0.0.1:54121)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.609501951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"io task","client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_connection connected from Ok(127.0.0.1:60065)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.60954149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"io task","client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.610337561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.61036663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.61038226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.610396589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.610410389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.610424048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.610984298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611063925Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611093544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611512699Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611541228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611555857Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611632725Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611651864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611665483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611677713Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611691132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[0]R flush_numbers[0..12]: [152, 155, 143, 155, 155, 155, 154, 147, 148, 155, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611709032Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[0]R generation[0..12]: [80, 82, 74, 82, 82, 82, 81, 76, 77, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611722911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611737181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[1]R flush_numbers[0..12]: [152, 155, 143, 155, 155, 155, 154, 147, 148, 155, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.61175102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[1]R generation[0..12]: [80, 82, 74, 82, 82, 82, 81, 76, 77, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.61176454Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611781449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[2]R flush_numbers[0..12]: [152, 155, 143, 155, 155, 155, 154, 147, 148, 155, 155, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611795609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[2]R generation[0..12]: [80, 82, 74, 82, 82, 82, 81, 76, 77, 82, 82, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611809418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611823578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611836667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611849877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Next flush: 157","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611862866Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611875906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","Wait for a query_work_queue command to finish before sending IO pid":1518,"":"downstairs","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611898465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611911104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"0","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611924494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"1","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611937503Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"client":"2","":"downstairs","session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611950663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611962883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"3cdcc472-c077-4f2f-b3da-29e0941515b4 is now active with session: 020c9af3-fab9-4340-836f-ffd96050c208","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611975692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.611989512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518,"session_id":"020c9af3-fab9-4340-836f-ffd96050c208"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:13.612002001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 461 len 12288 data: 12 11 10 02/30 Read block 115 len 40960 03/30 Write block 347 len 12288 data: 7 8 8 04/30 Write block 293 len 12288 data: 6 4 5 05/30 Write block 396 len 12288 data: 11 12 13 06/30 Flush 07/30 Write block 433 len 4096 data: 5 08/30 Write block 297 len 8192 data: 6 8 09/30 Read block 204 len 32768 10/30 Write block 38 len 12288 data: 10 9 8 11/30 Write block 215 len 8192 data: 6 9 12/30 Read block 141 len 16384 13/30 Read block 175 len 24576 14/30 Read block 377 len 4096 15/30 Write block 425 len 4096 data: 9 16/30 Write block 459 len 28672 data: 12 9 13 12 11 8 9 17/30 Write block 385 len 32768 data: 11 10 7 10 8 6 6 8 18/30 Read block 125 len 4096 19/30 Read block 58 len 40960 20/30 Write block 126 len 36864 data: 11 12 12 10 10 9 9 8 11 21/30 Read block 239 len 40960 22/30 Write block 120 len 24576 data: 10 8 7 7 7 9 23/30 Write block 102 len 16384 data: 7 7 8 8 24/30 Read block 260 len 8192 25/30 Read block 88 len 36864 26/30 Read block 21 len 24576 27/30 Read block 379 len 16384 28/30 Read block 386 len 20480 29/30 Write block 127 len 40960 data: 13 13 11 11 10 10 9 12 10 10 30/30 Write block 293 len 12288 data: 7 5 6 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 1 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Write 3 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 1 Done Done Done false 1014 Acked Write 1 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 8 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 10 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 6 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 5 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 22 22 22 66 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:13.631963601Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1518} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 82 82 82 155 155 155 T T T 2 040-059 74 74 74 143 143 143 T T T 3 060-079 82 82 82 155 155 155 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 82 82 82 155 155 155 T T T 6 120-139 81 81 81 154 154 154 T T T 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 82 82 82 155 155 155 F F F 10 200-219 82 82 82 155 155 155 T T T 11 220-239 82 82 82 155 155 155 F F F 12 240-259 82 82 82 155 155 155 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 84 84 84 157 157 157 T T T 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 84 84 84 157 157 157 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 84 84 84 157 157 157 T T T 20 400-419 83 83 83 156 156 156 F F F 21 420-439 83 83 83 156 156 156 T T T 22 440-459 82 82 82 155 155 155 T T T 23 460-479 84 84 84 157 157 157 T T T 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 84, Max flush: 157 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 85 85 85 158 158 158 F F F 2 040-059 85 85 85 158 158 158 F F F 3 060-079 82 82 82 155 155 155 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 85 85 85 158 158 158 F F F 6 120-139 85 85 85 158 158 158 F F F 7 140-159 76 76 76 147 147 147 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 82 82 82 155 155 155 F F F 10 200-219 85 85 85 158 158 158 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 82 82 82 155 155 155 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 85 85 85 158 158 158 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 84 84 84 157 157 157 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 85 85 85 158 158 158 F F F 20 400-419 83 83 83 156 156 156 F F F 21 420-439 85 85 85 158 158 158 F F F 22 440-459 85 85 85 158 158 158 F F F 23 460-479 85 85 85 158 158 158 F F F 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 85, Max flush: 158 Begin loop 43 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.014120969Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:18.055860363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.056252219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.086902392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.087433783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.087470612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.087485301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.087500181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} {"msg":"Crucible 95e536f5-4d35-4fbe-8f9e-ab37355540ae has session id: d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.087977403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Upstairs opts: Upstairs UUID: 95e536f5-4d35-4fbe-8f9e-ab37355540ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.088010762Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Crucible stats registered with UUID: 95e536f5-4d35-4fbe-8f9e-ab37355540ae","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.088025992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117793617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117825606Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} {"msg":"95e536f5-4d35-4fbe-8f9e-ab37355540ae active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117862405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117879204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117908793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117926732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117943942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117960581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.117977471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.11799395Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118011119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118028269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_connection connected from Ok(127.0.0.1:63515)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118308279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"io task","client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_connection connected from Ok(127.0.0.1:42675)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118357457Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"io task","client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_connection connected from Ok(127.0.0.1:48929)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118393936Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"io task","client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118864508Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118895367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118911327Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118925696Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118949175Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.118982094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.119564083Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.119606062Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.1196378Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120153512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120178361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120193491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120251018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120280767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120296667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120309526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120323556Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[0]R flush_numbers[0..12]: [152, 158, 158, 155, 155, 158, 158, 147, 148, 155, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120337785Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[0]R generation[0..12]: [80, 85, 85, 82, 82, 85, 85, 76, 77, 82, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120351775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120366074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[1]R flush_numbers[0..12]: [152, 158, 158, 155, 155, 158, 158, 147, 148, 155, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120383544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[1]R generation[0..12]: [80, 85, 85, 82, 82, 85, 85, 76, 77, 82, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120397773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120412032Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[2]R flush_numbers[0..12]: [152, 158, 158, 155, 155, 158, 158, 147, 148, 155, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120425972Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[2]R generation[0..12]: [80, 85, 85, 82, 82, 85, 85, 76, 77, 82, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120439611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120453831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120467101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.12048059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Next flush: 159","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.12049378Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120506889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"":"downstairs","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120520169Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120532328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526Wait for a query_work_queue command to finish before sending IO ,"client":"0","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120557817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"1","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120571117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"client":"2","":"downstairs","session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120586216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120599786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"95e536f5-4d35-4fbe-8f9e-ab37355540ae is now active with session: d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120612775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120625015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526,"session_id":"d17605f7-6ccb-4e7e-9bd1-3cd7ea7ec746"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:18.120637355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 429 len 16384 data: 4 3 3 5 02/30 Write block 69 len 20480 data: 6 7 7 5 3 03/30 Write block 50 len 20480 data: 7 5 5 6 7 04/30 Read block 477 len 4096 05/30 Read block 434 len 24576 06/30 Read block 591 len 12288 07/30 Write block 389 len 36864 data: 9 7 7 9 8 9 12 12 13 08/30 Read block 251 len 40960 09/30 Read block 241 len 28672 10/30 Read block 5 len 36864 11/30 Read block 232 len 36864 12/30 Read block 214 len 16384 13/30 Write block 144 len 16384 data: 5 7 7 7 14/30 Read block 405 len 20480 15/30 Write block 143 len 40960 data: 5 6 8 8 8 6 5 5 5 5 16/30 Write block 391 len 16384 data: 8 10 9 10 17/30 Read block 430 len 40960 18/30 Write block 242 len 12288 data: 7 7 10 19/30 Read block 158 len 24576 20/30 Write block 139 len 8192 data: 8 6 21/30 Read block 86 len 8192 22/30 Read block 63 len 36864 23/30 Read block 211 len 28672 24/30 Read block 161 len 36864 25/30 Read block 27 len 20480 26/30 Read block 581 len 16384 27/30 Read block 551 len 4096 28/30 Read block 481 len 24576 29/30 Write block 193 len 20480 data: 7 7 4 5 6 30/30 Read block 481 len 8192 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 3 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 10 Done Done Done false 1008 Acked Read 7 Done Done Done false 1009 Acked Read 9 Done Done Done false 1010 Acked Read 9 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Write 4 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 10 Done Done Done false 1015 Acked Write 4 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Write 3 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 2 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:18.140583824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1526} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 85 85 85 158 158 158 F F F 2 040-059 85 85 85 158 158 158 T T T 3 060-079 82 82 82 155 155 155 T T T 4 080-099 82 82 82 155 155 155 F F F 5 100-119 85 85 85 158 158 158 F F F 6 120-139 85 85 85 158 158 158 T T T 7 140-159 76 76 76 147 147 147 T T T 8 160-179 77 77 77 148 148 148 F F F 9 180-199 82 82 82 155 155 155 T T T 10 200-219 85 85 85 158 158 158 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 82 82 82 155 155 155 T T T 13 260-279 80 80 80 153 153 153 F F F 14 280-299 85 85 85 158 158 158 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 84 84 84 157 157 157 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 85 85 85 158 158 158 T T T 20 400-419 83 83 83 156 156 156 F F F 21 420-439 85 85 85 158 158 158 T T T 22 440-459 85 85 85 158 158 158 F F F 23 460-479 85 85 85 158 158 158 F F F 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 85, Max flush: 158 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 85 85 85 158 158 158 F F F 2 040-059 86 86 86 159 159 159 F F F 3 060-079 86 86 86 159 159 159 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 85 85 85 158 158 158 F F F 6 120-139 86 86 86 159 159 159 F F F 7 140-159 86 86 86 159 159 159 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 86 86 86 159 159 159 F F F 10 200-219 85 85 85 158 158 158 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 86 86 86 159 159 159 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 85 85 85 158 158 158 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 84 84 84 157 157 157 F F F 18 360-379 78 78 78 149 149 149 F F F 19 380-399 86 86 86 159 159 159 F F F 20 400-419 83 83 83 156 156 156 F F F 21 420-439 86 86 86 159 159 159 F F F 22 440-459 85 85 85 158 158 158 F F F 23 460-479 85 85 85 158 158 158 F F F 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 86, Max flush: 159 Begin loop 44 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.526330634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.570705442Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.57131957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.571353989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.571369188Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.571384577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} {"msg":"Crucible 6be5abed-41b5-4609-9185-50ff08e3b011 has session id: c27b4ec8-851e-4043-a131-e314bc20d5b9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.57186269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Upstairs opts: Upstairs UUID: 6be5abed-41b5-4609-9185-50ff08e3b011, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.571905129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Crucible stats registered with UUID: 6be5abed-41b5-4609-9185-50ff08e3b011","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.571930228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602100568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602134007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} {"msg":"6be5abed-41b5-4609-9185-50ff08e3b011 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602169016Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602192115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602220084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602238954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602255883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602272282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602289092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602305701Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.60232368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.6023411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:50668)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602681018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"io task","client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:54366)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602727606Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"io task","client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:60559)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.602759375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"io task","client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603320104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603348114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603363543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603377573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603394432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603408481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.603952552Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604097596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604257781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604582619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604618658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604645447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604663926Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604678026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604691545Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604710034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604723484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[0]R flush_numbers[0..12]: [152, 158, 159, 159, 155, 158, 159, 159, 148, 159, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604752443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[0]R generation[0..12]: [80, 85, 86, 86, 82, 85, 86, 86, 77, 86, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604767452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604786022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[1]R flush_numbers[0..12]: [152, 158, 159, 159, 155, 158, 159, 159, 148, 159, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604800301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[1]R generation[0..12]: [80, 85, 86, 86, 82, 85, 86, 86, 77, 86, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604813821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.60482781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[2]R flush_numbers[0..12]: [152, 158, 159, 159, 155, 158, 159, 159, 148, 159, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.60484176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[2]R generation[0..12]: [80, 85, 86, 86, 82, 85, 86, 86, 77, 86, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604855279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604869488Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Max found gen is 87","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604882568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Generation requested: 88 >= found:87","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604896068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604909427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604933536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"":"downstairs","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"No downstairs reconciliation required","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604955776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604968215Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"0","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604981384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"1","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.604996884Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"client":"2","":"downstairs","session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.605010254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.605022633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"6be5abed-41b5-4609-9185-50ff08e3b011 is now active with session: c27b4ec8-851e-4043-a131-e314bc20d5b9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.605035613Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.605047622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534,"session_id":"c27b4ec8-851e-4043-a131-e314bc20d5b9"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:22.605059902Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 394 len 12288 data: 11 13 13 02/30 Write block 118 len 36864 data: 9 11 11 9 8 8 8 10 12 03/30 Write block 147 len 4096 data: 9 04/30 Read block 430 len 8192 05/30 Write block 358 len 16384 data: 8 9 10 9 06/30 Read block 171 len 32768 07/30 Write block 186 len 4096 data: 7 08/30 Write block 376 len 20480 data: 8 8 7 9 10 09/30 Write block 295 len 16384 data: 7 6 7 9 10/30 Read block 462 len 24576 11/30 Flush 12/30 Write block 184 len 28672 data: 6 7 8 7 6 8 7 13/30 Flush 14/30 Write block 118 len 28672 data: 10 12 12 10 9 9 9 15/30 Read block 324 len 16384 16/30 Read block 428 len 28672 17/30 Read block 451 len 16384 18/30 Write block 418 len 24576 data: 6 6 9 9 8 12 19/30 Read block 85 len 36864 20/30 Write block 466 len 36864 data: 9 9 7 7 7 7 6 9 9 21/30 Write block 28 len 28672 data: 9 8 8 8 7 7 9 22/30 Read block 451 len 8192 23/30 Read block 286 len 8192 24/30 Write block 40 len 12288 data: 9 8 7 25/30 Read block 536 len 28672 26/30 Read block 562 len 8192 27/30 Read block 33 len 16384 28/30 Read block 472 len 24576 29/30 Write block 29 len 40960 data: 9 9 9 8 8 10 7 7 9 11 30/30 Read block 290 len 32768 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 7 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 7 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Write 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Done Done Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 2 3 Done 17 16 15 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:22.628035082Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1534} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 85 85 85 158 158 158 T T T 2 040-059 86 86 86 159 159 159 T T T 3 060-079 86 86 86 159 159 159 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 88 88 88 160 160 160 T T T 6 120-139 88 88 88 160 160 160 T T T 7 140-159 88 88 88 160 160 160 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 88 88 88 161 161 161 F F F 10 200-219 85 85 85 158 158 158 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 86 86 86 159 159 159 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 88 88 88 160 160 160 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 88 88 88 160 160 160 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 83 83 83 156 156 156 T T T 21 420-439 86 86 86 159 159 159 T T T 22 440-459 85 85 85 158 158 158 F F F 23 460-479 85 85 85 158 158 158 T T T 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 88, Max flush: 161 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 89 89 89 162 162 162 F F F 2 040-059 89 89 89 162 162 162 F F F 3 060-079 86 86 86 159 159 159 F F F 4 080-099 82 82 82 155 155 155 F F F 5 100-119 89 89 89 162 162 162 F F F 6 120-139 89 89 89 162 162 162 F F F 7 140-159 88 88 88 160 160 160 F F F 8 160-179 77 77 77 148 148 148 F F F 9 180-199 88 88 88 161 161 161 F F F 10 200-219 85 85 85 158 158 158 F F F 11 220-239 82 82 82 155 155 155 F F F 12 240-259 86 86 86 159 159 159 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 88 88 88 160 160 160 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 88 88 88 160 160 160 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 89 89 89 162 162 162 F F F 21 420-439 89 89 89 162 162 162 F F F 22 440-459 85 85 85 158 158 158 F F F 23 460-479 89 89 89 162 162 162 F F F 24 480-499 81 81 81 154 154 154 F F F 25 500-519 80 80 80 153 153 153 F F F 26 520-539 82 82 82 155 155 155 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 83 83 83 156 156 156 F F F 29 580-599 83 83 83 156 156 156 F F F Max gen: 89, Max flush: 162 Begin loop 45 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:26.964747471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:27.006939368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.007326604Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.038208069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.03873675Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.038780809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.038798498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.038816638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} {"msg":"Crucible c447f1c1-4d27-4a92-a3e6-18dbbd3f15e0 has session id: 80346cc6-9427-44fa-b46c-8ef01acbe5af","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.039263122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Upstairs opts: Upstairs UUID: c447f1c1-4d27-4a92-a3e6-18dbbd3f15e0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.03931272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Crucible stats registered with UUID: c447f1c1-4d27-4a92-a3e6-18dbbd3f15e0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.039341509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069711432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069742991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} {"msg":"c447f1c1-4d27-4a92-a3e6-18dbbd3f15e0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069821768Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069849357Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069877886Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069895676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.069959853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070000142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070021031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.07003888Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070064839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070099218Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_connection connected from Ok(127.0.0.1:37002)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070134407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"io task","client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_connection connected from Ok(127.0.0.1:59214)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070171976Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"io task","client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_connection connected from Ok(127.0.0.1:34462)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070210674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"io task","client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.07087924Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070910199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070937608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.070969977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.071001256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.071030265Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.071491828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.071520597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.071782377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072028899Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072057968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072072907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072097026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072111946Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072125315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072238471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.07225846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[0]R flush_numbers[0..12]: [152, 162, 162, 159, 155, 162, 162, 160, 148, 161, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.07227303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[0]R generation[0..12]: [80, 89, 89, 86, 82, 89, 89, 88, 77, 88, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072286949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072305269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[1]R flush_numbers[0..12]: [152, 162, 162, 159, 155, 162, 162, 160, 148, 161, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072319998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[1]R generation[0..12]: [80, 89, 89, 86, 82, 89, 89, 88, 77, 88, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072333778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072347667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[2]R flush_numbers[0..12]: [152, 162, 162, 159, 155, 162, 162, 160, 148, 161, 158, 155]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072361597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[2]R generation[0..12]: [80, 89, 89, 86, 82, 89, 89, 88, 77, 88, 85, 82]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072375176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072389346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072402265Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072415545Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Next flush: 163","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072428794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072441924Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072464173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072476472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"0","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072489952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"1","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072504931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"client":"2","":"downstairs","session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072518421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.07254565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"c447f1c1-4d27-4a92-a3e6-18dbbd3f15e0 is now active with session: 80346cc6-9427-44fa-b46c-8ef01acbe5af","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072559029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072571139Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542,"session_id":"80346cc6-9427-44fa-b46c-8ef01acbe5af"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:27.072583399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 20480 data: 6 8 8 8 7 02/30 Read block 347 len 28672 03/30 Write block 577 len 16384 data: 6 6 5 5 04/30 Write block 523 len 16384 data: 7 7 7 7 05/30 Read block 72 len 36864 06/30 Read block 523 len 20480 07/30 Write block 483 len 12288 data: 6 8 8 08/30 Read block 428 len 8192 09/30 Read block 508 len 28672 10/30 Flush 11/30 Write block 152 len 4096 data: 6 12/30 Write block 447 len 36864 data: 6 5 6 6 4 6 10 8 8 13/30 Write block 185 len 12288 data: 8 9 8 14/30 Write block 50 len 28672 data: 8 6 6 7 8 6 6 15/30 Write block 75 len 24576 data: 6 7 5 4 6 6 16/30 Read block 61 len 20480 17/30 Read block 384 len 16384 18/30 Write block 200 len 28672 data: 3 4 7 8 10 10 10 19/30 Read block 48 len 8192 20/30 Write block 102 len 28672 data: 8 8 9 9 7 5 6 21/30 Read block 266 len 12288 22/30 Write block 23 len 32768 data: 6 7 10 9 9 10 10 10 23/30 Write block 513 len 28672 data: 6 7 6 7 9 8 7 24/30 Read block 350 len 20480 25/30 Flush 26/30 Write block 33 len 40960 data: 9 11 8 8 10 12 10 10 9 8 27/30 Write block 171 len 8192 data: 5 5 28/30 Write block 20 len 36864 data: 10 9 10 7 8 11 10 10 11 29/30 Read block 406 len 36864 30/30 Write block 216 len 20480 data: 10 10 8 8 7 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 10 Done Done Done false 1026 Acked Write 2 Done Done Done false 1027 Acked Write 9 Done Sent Sent false 1028 Acked Read 9 Done Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 3 7 Done 4 2 2 8 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:27.096396589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1542} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 90 90 90 164 164 164 T T T 2 040-059 90 90 90 164 164 164 T T T 3 060-079 90 90 90 164 164 164 F F F 4 080-099 90 90 90 164 164 164 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 89 89 89 162 162 162 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 77 77 77 148 148 148 T T T 9 180-199 90 90 90 164 164 164 F F F 10 200-219 90 90 90 164 164 164 T T T 11 220-239 82 82 82 155 155 155 T T T 12 240-259 86 86 86 159 159 159 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 88 88 88 160 160 160 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 90 90 90 163 163 163 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 89 89 89 162 162 162 F F F 21 420-439 89 89 89 162 162 162 F F F 22 440-459 90 90 90 164 164 164 F F F 23 460-479 89 89 89 162 162 162 F F F 24 480-499 90 90 90 163 163 163 F F F 25 500-519 90 90 90 164 164 164 F F F 26 520-539 90 90 90 163 163 163 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 90 90 90 163 163 163 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 90, Max flush: 164 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 91 91 91 165 165 165 F F F 2 040-059 91 91 91 165 165 165 F F F 3 060-079 90 90 90 164 164 164 F F F 4 080-099 90 90 90 164 164 164 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 89 89 89 162 162 162 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 91 91 91 165 165 165 F F F 9 180-199 90 90 90 164 164 164 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 86 86 86 159 159 159 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 88 88 88 160 160 160 F F F 15 300-319 75 75 75 144 144 144 F F F 16 320-339 82 82 82 155 155 155 F F F 17 340-359 90 90 90 163 163 163 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 89 89 89 162 162 162 F F F 21 420-439 89 89 89 162 162 162 F F F 22 440-459 90 90 90 164 164 164 F F F 23 460-479 89 89 89 162 162 162 F F F 24 480-499 90 90 90 163 163 163 F F F 25 500-519 90 90 90 164 164 164 F F F 26 520-539 90 90 90 163 163 163 F F F 27 540-559 82 82 82 155 155 155 F F F 28 560-579 90 90 90 163 163 163 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 91, Max flush: 165 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.439667222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:31.481667866Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.482078371Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.513026634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.513599823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.513635862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.513650041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.513664461Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} {"msg":"Crucible 3d847e32-ca85-4454-a541-9d04a95051b8 has session id: 2cd4e56a-db06-45ec-bd9b-7442c24d71c7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.514091226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Upstairs opts: Upstairs UUID: 3d847e32-ca85-4454-a541-9d04a95051b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.514125384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Crucible stats registered with UUID: 3d847e32-ca85-4454-a541-9d04a95051b8","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.514141534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.543971427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544007095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} {"msg":"3d847e32-ca85-4454-a541-9d04a95051b8 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544050804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544070123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544098512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544120402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54415245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54417015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544186779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544203548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544221208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544238247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55737)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544372552Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"io task","client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:44063)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544421451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"io task","client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:49429)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.544460949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"io task","client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545151244Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545180663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545199532Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545226202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54525497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545287599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54581732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545851219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.545880558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54636243Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54638798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546403629Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546475066Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546494566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546509525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546522015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546535474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[0]R flush_numbers[0..12]: [152, 165, 165, 164, 164, 164, 162, 164, 165, 164, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546549774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[0]R generation[0..12]: [80, 91, 91, 90, 90, 90, 89, 90, 91, 90, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546563613Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546577843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[1]R flush_numbers[0..12]: [152, 165, 165, 164, 164, 164, 162, 164, 165, 164, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546595712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[1]R generation[0..12]: [80, 91, 91, 90, 90, 90, 89, 90, 91, 90, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546610012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546624361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[2]R flush_numbers[0..12]: [152, 165, 165, 164, 164, 164, 162, 164, 165, 164, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546638171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[2]R generation[0..12]: [80, 91, 91, 90, 90, 90, 89, 90, 91, 90, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54665168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.54666553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546678749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546740627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Next flush: 166","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546771926Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546788145Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"":"downstairs","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546802315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546815184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"0","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546828944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"1","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546842423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"client":"2","":"downstairs","session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546855873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546871712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"3d847e32-ca85-4454-a541-9d04a95051b8 is now active with session: 2cd4e56a-db06-45ec-bd9b-7442c24d71c7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546885242Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546897551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550,"session_id":"2cd4e56a-db06-45ec-bd9b-7442c24d71c7"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:31.546910001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 420 len 16384 data: 10 10 9 13 02/30 Read block 359 len 8192 03/30 Write block 163 len 24576 data: 7 7 8 6 6 5 04/30 Write block 285 len 20480 data: 13 14 13 12 10 05/30 Read block 406 len 20480 06/30 Write block 564 len 8192 data: 4 8 07/30 Read block 350 len 12288 08/30 Read block 292 len 24576 09/30 Write block 409 len 24576 data: 8 8 7 6 6 6 10/30 Read block 381 len 40960 11/30 Read block 560 len 8192 12/30 Read block 476 len 24576 13/30 Write block 424 len 40960 data: 8 10 8 8 5 5 4 4 6 6 14/30 Write block 194 len 16384 data: 8 5 6 7 15/30 Write block 126 len 12288 data: 13 14 14 16/30 Write block 283 len 24576 data: 17 13 14 15 14 13 17/30 Write block 546 len 40960 data: 10 11 14 15 14 13 14 14 9 8 18/30 Write block 337 len 32768 data: 10 10 6 5 7 9 9 9 19/30 Read block 12 len 24576 20/30 Read block 313 len 4096 21/30 Write block 253 len 20480 data: 12 9 11 11 8 22/30 Write block 313 len 12288 data: 5 5 3 23/30 Read block 211 len 20480 24/30 Read block 492 len 16384 25/30 Write block 531 len 20480 data: 8 8 7 7 6 26/30 Flush 27/30 Write block 338 len 8192 data: 11 7 28/30 Write block 86 len 20480 data: 9 10 11 10 12 29/30 Read block 426 len 20480 30/30 Read block 517 len 20480 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 2 Done Done Done false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 3 3 Done 4 4 1 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:31.567654402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1550} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 91 91 91 165 165 165 F F F 2 040-059 91 91 91 165 165 165 F F F 3 060-079 90 90 90 164 164 164 F F F 4 080-099 90 90 90 164 164 164 T T T 5 100-119 90 90 90 164 164 164 F F F 6 120-139 92 92 92 166 166 166 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 92 92 92 166 166 166 F F F 9 180-199 92 92 92 166 166 166 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 92 92 92 166 166 166 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 92 92 92 166 166 166 F F F 15 300-319 92 92 92 166 166 166 F F F 16 320-339 92 92 92 166 166 166 T T T 17 340-359 92 92 92 166 166 166 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 92 92 92 166 166 166 F F F 21 420-439 92 92 92 166 166 166 F F F 22 440-459 90 90 90 164 164 164 F F F 23 460-479 89 89 89 162 162 162 F F F 24 480-499 90 90 90 163 163 163 F F F 25 500-519 90 90 90 164 164 164 F F F 26 520-539 92 92 92 166 166 166 F F F 27 540-559 92 92 92 166 166 166 F F F 28 560-579 92 92 92 166 166 166 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 92, Max flush: 166 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 91 91 91 165 165 165 F F F 2 040-059 91 91 91 165 165 165 F F F 3 060-079 90 90 90 164 164 164 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 92 92 92 166 166 166 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 92 92 92 166 166 166 F F F 9 180-199 92 92 92 166 166 166 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 92 92 92 166 166 166 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 92 92 92 166 166 166 F F F 15 300-319 92 92 92 166 166 166 F F F 16 320-339 93 93 93 167 167 167 F F F 17 340-359 92 92 92 166 166 166 F F F 18 360-379 88 88 88 160 160 160 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 92 92 92 166 166 166 F F F 21 420-439 92 92 92 166 166 166 F F F 22 440-459 90 90 90 164 164 164 F F F 23 460-479 89 89 89 162 162 162 F F F 24 480-499 90 90 90 163 163 163 F F F 25 500-519 90 90 90 164 164 164 F F F 26 520-539 92 92 92 166 166 166 F F F 27 540-559 92 92 92 166 166 166 F F F 28 560-579 92 92 92 166 166 166 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 93, Max flush: 167 Begin loop 47 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.875491034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.918987243Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.919578252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.919614571Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.91963334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.91964864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} {"msg":"Crucible 375b785c-3dcb-4555-b015-eab9ebe24efd has session id: 183f94a2-36ce-4183-8ba8-9bad61c18eac","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.920093124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Upstairs opts: Upstairs UUID: 375b785c-3dcb-4555-b015-eab9ebe24efd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.920137292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Crucible stats registered with UUID: 375b785c-3dcb-4555-b015-eab9ebe24efd","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.920157781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951717142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.95176431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} {"msg":"375b785c-3dcb-4555-b015-eab9ebe24efd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.9517829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951799609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951826788Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951844408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951861237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951877726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951900526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951917995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951935434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.951952744Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:53313)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.952106038Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"io task","client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:50998)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.952410677Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"io task","client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_connection connected from Ok(127.0.0.1:52396)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.952451675Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"io task","client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.95287125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.952910569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953019445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953062153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953094982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953123411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953554506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953628453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.953664092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954156474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954185463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954200523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954213282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954226862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954240101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.95425399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.95426767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[0]R flush_numbers[0..12]: [152, 165, 165, 164, 167, 164, 166, 164, 166, 166, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954297349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[0]R generation[0..12]: [80, 91, 91, 90, 93, 90, 92, 90, 92, 92, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954312028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954326048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[1]R flush_numbers[0..12]: [152, 165, 165, 164, 167, 164, 166, 164, 166, 166, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954339997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[1]R generation[0..12]: [80, 91, 91, 90, 93, 90, 92, 90, 92, 92, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954353717Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954367696Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[2]R flush_numbers[0..12]: [152, 165, 165, 164, 167, 164, 166, 164, 166, 166, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954385526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[2]R generation[0..12]: [80, 91, 91, 90, 93, 90, 92, 90, 92, 92, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954399415Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954413265Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954426334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954439674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Next flush: 168","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954452773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954465653Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"":"downstairs","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:06:35.954478932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954497422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"0","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954510831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"1","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954524041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"client":"2","":"downstairs","session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.95453748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.95455014Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"375b785c-3dcb-4555-b015-eab9ebe24efd is now active with session: 183f94a2-36ce-4183-8ba8-9bad61c18eac","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954563109Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954575029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558,"session_id":"183f94a2-36ce-4183-8ba8-9bad61c18eac"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:35.954587229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 496 len 36864 data: 9 9 7 9 9 8 9 10 10 02/30 Write block 128 len 20480 data: 15 12 12 11 11 03/30 Write block 66 len 12288 data: 7 6 5 04/30 Read block 200 len 36864 05/30 Read block 200 len 16384 06/30 Write block 360 len 32768 data: 11 10 11 12 10 7 7 7 07/30 Write block 436 len 28672 data: 6 6 6 5 5 5 3 08/30 Write block 173 len 16384 data: 4 4 5 5 09/30 Read block 440 len 12288 10/30 Read block 452 len 16384 11/30 Write block 166 len 12288 data: 7 7 6 12/30 Read block 534 len 24576 13/30 Write block 457 len 36864 data: 9 10 13 10 14 13 12 9 10 14/30 Write block 31 len 8192 data: 10 9 15/30 Read block 87 len 36864 16/30 Flush 17/30 Write block 295 len 20480 data: 8 7 8 10 8 18/30 Read block 266 len 40960 19/30 Read block 82 len 40960 20/30 Flush 21/30 Write block 24 len 32768 data: 9 12 11 11 12 11 11 11 22/30 Write block 243 len 16384 data: 8 11 11 10 23/30 Write block 328 len 24576 data: 5 6 8 9 10 11 24/30 Write block 369 len 32768 data: 11 13 13 13 12 11 11 9 25/30 Flush 26/30 Write block 508 len 28672 data: 8 7 10 10 9 7 8 27/30 Read block 45 len 20480 28/30 Write block 492 len 32768 data: 8 7 10 8 10 10 8 10 29/30 Write block 553 len 4096 data: 15 30/30 Write block 474 len 8192 data: 10 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 7 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:35.978682359Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1558} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 94 94 94 170 170 170 F F F 2 040-059 91 91 91 165 165 165 F F F 3 060-079 94 94 94 168 168 168 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 94 94 94 168 168 168 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 94 94 94 168 168 168 F F F 9 180-199 92 92 92 166 166 166 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 94 94 94 170 170 170 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 94 94 94 169 169 169 F F F 15 300-319 92 92 92 166 166 166 F F F 16 320-339 94 94 94 170 170 170 F F F 17 340-359 92 92 92 166 166 166 F F F 18 360-379 94 94 94 170 170 170 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 92 92 92 166 166 166 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 94 94 94 168 168 168 F F F 23 460-479 94 94 94 168 168 168 T T F <--- 24 480-499 94 94 94 168 168 168 T T T 25 500-519 94 94 94 168 168 168 T T T 26 520-539 92 92 92 166 166 166 F F F 27 540-559 92 92 92 166 166 166 T T T 28 560-579 92 92 92 166 166 166 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 94, Max flush: 170 Error: Difference in extent metadata found! A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 94 94 94 170 170 170 F F F 2 040-059 91 91 91 165 165 165 F F F 3 060-079 94 94 94 168 168 168 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 94 94 94 168 168 168 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 94 94 94 168 168 168 F F F 9 180-199 92 92 92 166 166 166 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 94 94 94 170 170 170 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 94 94 94 169 169 169 F F F 15 300-319 92 92 92 166 166 166 F F F 16 320-339 94 94 94 170 170 170 F F F 17 340-359 92 92 92 166 166 166 F F F 18 360-379 94 94 94 170 170 170 F F F 19 380-399 88 88 88 160 160 160 F F F 20 400-419 92 92 92 166 166 166 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 94 94 94 168 168 168 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 95 95 95 171 171 171 F F F 26 520-539 92 92 92 166 166 166 F F F 27 540-559 95 95 95 171 171 171 F F F 28 560-579 92 92 92 166 166 166 F F F 29 580-599 90 90 90 163 163 163 F F F Max gen: 95, Max flush: 171 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.277229536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:40.320487514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.320854731Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.352543357Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.353231702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.353267081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.35328519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.35330012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} {"msg":"Crucible 944192ae-025f-4010-ab58-e9116d780ce5 has session id: e0c7582e-41b8-453d-9ec8-1347d5c240b4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.353703115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Upstairs opts: Upstairs UUID: 944192ae-025f-4010-ab58-e9116d780ce5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.353745714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Crucible stats registered with UUID: 944192ae-025f-4010-ab58-e9116d780ce5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.353770763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384262322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384300491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} {"msg":"944192ae-025f-4010-ab58-e9116d780ce5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384364328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384410727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384446026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384481284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384514973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384547532Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384580331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384618839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384653898Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:38406)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384770514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"io task","client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:49317)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.384836142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"io task","client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_connection connected from Ok(127.0.0.1:41136)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.38487461Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"io task","client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.385481548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.385578135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.385596284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.385610374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.385624203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.385650382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386113336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386173853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386410215Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386724563Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386758702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386775152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386787861Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386801871Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.38681526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386914936Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386935566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[0]R flush_numbers[0..12]: [152, 170, 165, 168, 167, 164, 168, 164, 168, 166, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386950235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[0]R generation[0..12]: [80, 94, 91, 94, 93, 90, 94, 90, 94, 92, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386964275Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386978404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[1]R flush_numbers[0..12]: [152, 170, 165, 168, 167, 164, 168, 164, 168, 166, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.386992304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[1]R generation[0..12]: [80, 94, 91, 94, 93, 90, 94, 90, 94, 92, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387005833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387020113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[2]R flush_numbers[0..12]: [152, 170, 165, 168, 167, 164, 168, 164, 168, 166, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387033952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[2]R generation[0..12]: [80, 94, 91, 94, 93, 90, 94, 90, 94, 92, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387050112Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387064591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387077931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.38709115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Next flush: 172","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.38710415Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387117339Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"":"downstairs","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387229115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387262234Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"0","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387292013Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"1","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387320962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"client":"2","":"downstairs","session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387350531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.38738775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"944192ae-025f-4010-ab58-e9116d780ce5 is now active with session: e0c7582e-41b8-453d-9ec8-1347d5c240b4"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387419938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387446017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566,"session_id":"e0c7582e-41b8-453d-9ec8-1347d5c240b4"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:40.387460917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 182 len 28672 data: 8 8 7 9 10 9 7 03/30 Read block 410 len 12288 04/30 Read block 528 len 40960 05/30 Write block 34 len 32768 data: 12 9 9 11 13 11 11 10 06/30 Write block 446 len 24576 data: 5 7 6 7 7 5 07/30 Read block 577 len 8192 08/30 Read block 558 len 8192 09/30 Write block 52 len 12288 data: 7 8 9 10/30 Flush 11/30 Write block 371 len 36864 data: 14 14 13 12 12 10 9 8 10 12/30 Write block 594 len 20480 data: 6 4 4 4 4 13/30 Read block 384 len 4096 14/30 Read block 120 len 32768 15/30 Write block 376 len 28672 data: 11 10 9 11 11 11 10 16/30 Read block 536 len 16384 17/30 Read block 127 len 4096 18/30 Read block 106 len 8192 19/30 Flush 20/30 Write block 332 len 28672 data: 11 12 11 12 12 11 12 21/30 Read block 421 len 20480 22/30 Flush 23/30 Write block 255 len 12288 data: 12 12 9 24/30 Read block 541 len 8192 25/30 Write block 326 len 12288 data: 4 5 6 26/30 Flush 27/30 Write block 161 len 32768 data: 7 9 8 8 9 8 8 7 28/30 Write block 131 len 32768 data: 12 12 10 13 11 11 10 7 29/30 Read block 233 len 28672 30/30 Write block 410 len 16384 data: 9 8 7 7 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 8 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:40.41395012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1566} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 96 96 96 173 173 173 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 94 94 94 168 168 168 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 94 94 94 168 168 168 T T T 7 140-159 90 90 90 164 164 164 F F F 8 160-179 94 94 94 168 168 168 T T T 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 96 96 96 176 176 176 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 94 94 94 169 169 169 F F F 15 300-319 92 92 92 166 166 166 F F F 16 320-339 96 96 96 176 176 176 F F F 17 340-359 92 92 92 166 166 166 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 92 92 92 166 166 166 T T T 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 95 95 95 171 171 171 F F F 26 520-539 92 92 92 166 166 166 F F F 27 540-559 95 95 95 171 171 171 F F F 28 560-579 92 92 92 166 166 166 F F F 29 580-599 96 96 96 174 174 174 F F F Max gen: 96, Max flush: 176 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 F F F 1 020-039 96 96 96 173 173 173 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 94 94 94 168 168 168 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 90 90 90 164 164 164 F F F 6 120-139 97 97 97 177 177 177 F F F 7 140-159 90 90 90 164 164 164 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 96 96 96 176 176 176 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 94 94 94 169 169 169 F F F 15 300-319 92 92 92 166 166 166 F F F 16 320-339 96 96 96 176 176 176 F F F 17 340-359 92 92 92 166 166 166 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 95 95 95 171 171 171 F F F 26 520-539 92 92 92 166 166 166 F F F 27 540-559 95 95 95 171 171 171 F F F 28 560-579 92 92 92 166 166 166 F F F 29 580-599 96 96 96 174 174 174 F F F Max gen: 97, Max flush: 177 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.729207765Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.772609898Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773204417Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773239186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773253425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773267854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} {"msg":"Crucible f97d6bc3-4c4c-490b-bf13-5cb700d3527f has session id: cdbad39b-4f51-4a52-bed9-331f407f48e1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773693009Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Upstairs opts: Upstairs UUID: f97d6bc3-4c4c-490b-bf13-5cb700d3527f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773734568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Crucible stats registered with UUID: f97d6bc3-4c4c-490b-bf13-5cb700d3527f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.773762467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.804797196Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.804830135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} {"msg":"f97d6bc3-4c4c-490b-bf13-5cb700d3527f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.804899973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.804922142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.804949231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.8049669Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.80498451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805076696Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805097465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805116865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805143994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805172513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:54187)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805391445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"io task","client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47582)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805437393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"io task","client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_connection connected from Ok(127.0.0.1:52487)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.805488591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"io task","client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806044381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.80608652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806118529Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806149967Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806180046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806207756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806677559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806811504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.806843322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807219299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807284817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807321355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807343135Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807358384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807376363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807389193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807402792Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[0]R flush_numbers[0..12]: [152, 173, 173, 168, 167, 164, 177, 164, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807423292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[0]R generation[0..12]: [80, 96, 96, 94, 93, 90, 97, 90, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807437671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.8074518Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[1]R flush_numbers[0..12]: [152, 173, 173, 168, 167, 164, 177, 164, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.80746583Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[1]R generation[0..12]: [80, 96, 96, 94, 93, 90, 97, 90, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807479659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807493789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[2]R flush_numbers[0..12]: [152, 173, 173, 168, 167, 164, 177, 164, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807507778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[2]R generation[0..12]: [80, 96, 96, 94, 93, 90, 97, 90, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807521468Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807535348Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807548377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807561597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Next flush: 178","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807576566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807589816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"":"downstairs","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807603015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"Wait for a query_work_queue command to finish before sending IO cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807623254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"0","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807636584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"1","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807649664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"client":"2","":"downstairs","session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807663123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807675403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"f97d6bc3-4c4c-490b-bf13-5cb700d3527f is now active with session: cdbad39b-4f51-4a52-bed9-331f407f48e1","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807688272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807700342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574,"session_id":"cdbad39b-4f51-4a52-bed9-331f407f48e1"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:44.807712621Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 146 len 8192 data: 9 10 02/30 Flush 03/30 Write block 111 len 12288 data: 6 8 9 04/30 Write block 148 len 40960 data: 7 6 6 6 7 4 4 6 7 8 05/30 Read block 501 len 4096 06/30 Write block 19 len 32768 data: 10 11 10 11 8 10 13 12 07/30 Read block 54 len 36864 08/30 Read block 244 len 36864 09/30 Read block 34 len 16384 10/30 Write block 303 len 4096 data: 8 11/30 Read block 199 len 32768 12/30 Write block 102 len 28672 data: 9 9 10 10 8 6 7 13/30 Read block 491 len 16384 14/30 Write block 560 len 28672 data: 5 5 5 4 5 9 8 15/30 Read block 557 len 32768 16/30 Write block 581 len 8192 data: 4 5 17/30 Read block 57 len 40960 18/30 Read block 90 len 24576 19/30 Write block 105 len 32768 data: 11 9 7 8 6 6 7 9 20/30 Write block 566 len 40960 data: 9 11 9 10 10 10 7 6 5 5 21/30 Read block 270 len 20480 22/30 Read block 321 len 24576 23/30 Write block 31 len 4096 data: 12 24/30 Read block 335 len 40960 25/30 Read block 412 len 32768 26/30 Write block 520 len 28672 data: 8 9 8 8 8 8 8 27/30 Write block 341 len 36864 data: 8 10 10 10 8 7 8 9 9 28/30 Write block 511 len 4096 data: 11 29/30 Read block 255 len 20480 30/30 Read block 527 len 16384 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 3 Done Done Done false 1003 Acked Write 10 Done Done Done false 1004 Acked Read 1 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Read 9 Done Done Done false 1007 Acked Read 9 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Read 8 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 2 Done Done Done false 1016 Acked Read 10 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 8 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 6 Done Done Done false 1022 Acked Write 1 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 7 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 4 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 28 27 28 83 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1001 1001 1001 Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:44.828720513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1574} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 80 80 80 152 152 152 T T T 1 020-039 96 96 96 173 173 173 T T T 2 040-059 96 96 96 173 173 173 F F F 3 060-079 94 94 94 168 168 168 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 90 90 90 164 164 164 T T T 6 120-139 97 97 97 177 177 177 F F F 7 140-159 98 98 98 178 178 178 T T T 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 96 96 96 176 176 176 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 94 94 94 169 169 169 F F F 15 300-319 92 92 92 166 166 166 T T T 16 320-339 96 96 96 176 176 176 F F F 17 340-359 92 92 92 166 166 166 T T T 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 95 95 95 171 171 171 T T T 26 520-539 92 92 92 166 166 166 T T T 27 540-559 95 95 95 171 171 171 F F F 28 560-579 92 92 92 166 166 166 T T T 29 580-599 96 96 96 174 174 174 T T T Max gen: 98, Max flush: 178 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 179 179 179 F F F 1 020-039 99 99 99 179 179 179 F F F 2 040-059 96 96 96 173 173 173 F F F 3 060-079 94 94 94 168 168 168 F F F 4 080-099 93 93 93 167 167 167 F F F 5 100-119 99 99 99 179 179 179 F F F 6 120-139 97 97 97 177 177 177 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 96 96 96 176 176 176 F F F 13 260-279 80 80 80 153 153 153 F F F 14 280-299 94 94 94 169 169 169 F F F 15 300-319 99 99 99 179 179 179 F F F 16 320-339 96 96 96 176 176 176 F F F 17 340-359 99 99 99 179 179 179 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 99 99 99 179 179 179 F F F 26 520-539 99 99 99 179 179 179 F F F 27 540-559 95 95 95 171 171 171 F F F 28 560-579 99 99 99 179 179 179 F F F 29 580-599 99 99 99 179 179 179 F F F Max gen: 99, Max flush: 179 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.206409623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:49.248481064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.24887326Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.279548253Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.280156081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.28019907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.280216499Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.280239748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} {"msg":"Crucible ef500b43-0cfa-4241-ad11-057d2be1989d has session id: 0f298a25-8472-460c-81d9-ec2f5be196ab","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.280622264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Upstairs opts: Upstairs UUID: ef500b43-0cfa-4241-ad11-057d2be1989d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.280671402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Crucible stats registered with UUID: ef500b43-0cfa-4241-ad11-057d2be1989d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.280702271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310285004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310318722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} {"msg":"ef500b43-0cfa-4241-ad11-057d2be1989d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31036981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31038973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310445458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310484286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310521975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310546884Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310568093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310608342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31064358Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310682429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:55338)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310806715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"io task","client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:56400)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.310852043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"io task","client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_connection connected from Ok(127.0.0.1:35255)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31092936Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"io task","client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.311620995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.311659174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.311690393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.311721392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.311747671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31176258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312332759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312364488Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312381418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312912169Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312941377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312959517Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312972727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.312986666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313000046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313064993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313084942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[0]R flush_numbers[0..12]: [179, 179, 173, 168, 167, 179, 177, 179, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313099372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[0]R generation[0..12]: [99, 99, 96, 94, 93, 99, 97, 99, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313113481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313127641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[1]R flush_numbers[0..12]: [179, 179, 173, 168, 167, 179, 177, 179, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31314181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[1]R generation[0..12]: [99, 99, 96, 94, 93, 99, 97, 99, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31315584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313170019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[2]R flush_numbers[0..12]: [179, 179, 173, 168, 167, 179, 177, 179, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313183989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[2]R generation[0..12]: [99, 99, 96, 94, 93, 99, 97, 99, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313197888Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313212008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313227967Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313241627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Next flush: 180","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313254876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313268036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66"Wait for a query_work_queue command to finish before sending IO ,"pid":1582,"":"downstairs","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313333634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313356842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"0","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313371852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"1","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313386081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"client":"2","":"downstairs","session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313399901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313412671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"ef500b43-0cfa-4241-ad11-057d2be1989d is now active with session: 0f298a25-8472-460c-81d9-ec2f5be196ab","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31342577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.31343909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582,"session_id":"0f298a25-8472-460c-81d9-ec2f5be196ab"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:49.313452119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 583 len 28672 data: 8 9 10 10 11 12 12 02/30 Read block 289 len 12288 03/30 Write block 543 len 4096 data: 4 04/30 Write block 284 len 24576 data: 14 15 16 15 14 11 05/30 Read block 247 len 12288 06/30 Read block 373 len 28672 07/30 Read block 36 len 32768 08/30 Read block 506 len 20480 09/30 Read block 313 len 36864 10/30 Read block 223 len 8192 11/30 Write block 570 len 16384 data: 11 11 8 7 12/30 Read block 59 len 28672 13/30 Read block 159 len 8192 14/30 Read block 328 len 28672 15/30 Read block 467 len 28672 16/30 Read block 200 len 8192 17/30 Flush 18/30 Write block 275 len 40960 data: 10 9 10 10 8 10 11 13 18 15 19/30 Write block 530 len 8192 data: 6 9 20/30 Read block 509 len 24576 21/30 Write block 52 len 12288 data: 8 9 10 22/30 Flush 23/30 Write block 101 len 24576 data: 6 10 10 11 12 10 24/30 Flush 25/30 Write block 41 len 32768 data: 11 9 8 7 7 7 7 6 26/30 Write block 331 len 40960 data: 10 12 13 12 13 13 12 13 8 6 27/30 Read block 479 len 36864 28/30 Write block 74 len 36864 data: 5 7 8 6 5 7 7 6 6 29/30 Read block 224 len 24576 30/30 Read block 575 len 36864 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 8 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 6 5 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:49.338796384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1582} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 179 179 179 F F F 1 020-039 99 99 99 179 179 179 F F F 2 040-059 100 100 100 181 181 181 T T T 3 060-079 94 94 94 168 168 168 T T T 4 080-099 93 93 93 167 167 167 T T T 5 100-119 100 100 100 182 182 182 F F F 6 120-139 97 97 97 177 177 177 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 96 96 96 176 176 176 F F F 13 260-279 100 100 100 181 181 181 F F F 14 280-299 100 100 100 181 181 181 F F F 15 300-319 99 99 99 179 179 179 F F F 16 320-339 96 96 96 176 176 176 T T T 17 340-359 99 99 99 179 179 179 T T T 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 99 99 99 179 179 179 F F F 26 520-539 100 100 100 181 181 181 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 100 100 100 180 180 180 F F F 29 580-599 100 100 100 180 180 180 F F F Max gen: 100, Max flush: 182 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 99 99 99 179 179 179 F F F 1 020-039 99 99 99 179 179 179 F F F 2 040-059 101 101 101 183 183 183 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 101 101 101 183 183 183 F F F 5 100-119 100 100 100 182 182 182 F F F 6 120-139 97 97 97 177 177 177 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 F F F 11 220-239 91 91 91 165 165 165 F F F 12 240-259 96 96 96 176 176 176 F F F 13 260-279 100 100 100 181 181 181 F F F 14 280-299 100 100 100 181 181 181 F F F 15 300-319 99 99 99 179 179 179 F F F 16 320-339 101 101 101 183 183 183 F F F 17 340-359 101 101 101 183 183 183 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 99 99 99 179 179 179 F F F 26 520-539 100 100 100 181 181 181 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 100 100 100 180 180 180 F F F 29 580-599 100 100 100 180 180 180 F F F Max gen: 101, Max flush: 183 Begin loop 51 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.679240759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:53.721090828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.721497184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.75233907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.752878651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.752913829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.752927299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.752942528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} {"msg":"Crucible 72e6b587-d5ac-4099-8d17-ce18e396d16e has session id: 5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.753373483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Upstairs opts: Upstairs UUID: 72e6b587-d5ac-4099-8d17-ce18e396d16e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.753419601Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Crucible stats registered with UUID: 72e6b587-d5ac-4099-8d17-ce18e396d16e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.75344557Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783164237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783207615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} {"msg":"72e6b587-d5ac-4099-8d17-ce18e396d16e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783237124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783261723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783308401Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.78334478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783380799Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783414668Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783502844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783550423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783589171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.78362667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_connection connected from Ok(127.0.0.1:65502)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783693778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"io task","client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_connection connected from Ok(127.0.0.1:49411)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783757015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"io task","client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_connection connected from Ok(127.0.0.1:48550)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.783798794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"io task","client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.784230958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.784259157Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.784367033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.784435301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.784620824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.78472582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.784865865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785090537Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785224372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785433915Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785475303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785504312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785532631Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.78556301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785588909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785739993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785776632Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[0]R flush_numbers[0..12]: [179, 179, 183, 183, 183, 182, 177, 179, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785803431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[0]R generation[0..12]: [99, 99, 101, 101, 101, 100, 97, 99, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.78582935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785855799Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[1]R flush_numbers[0..12]: [179, 179, 183, 183, 183, 182, 177, 179, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785880839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[1]R generation[0..12]: [99, 99, 101, 101, 101, 100, 97, 99, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785905507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785932427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[2]R flush_numbers[0..12]: [179, 179, 183, 183, 183, 182, 177, 179, 177, 173, 165, 165]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785960665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[2]R generation[0..12]: [99, 99, 101, 101, 101, 100, 97, 99, 97, 96, 91, 91]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.785985695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786014784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,""Wait for a query_work_queue command to finish before sending IO :"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786052202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786088771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Next flush: 184","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.78611901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786147219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"":"downstairs","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786175688Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786213496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"0","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786245355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"1","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786275994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"client":"2","":"downstairs","session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786293473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786306423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"72e6b587-d5ac-4099-8d17-ce18e396d16e is now active with session: 5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786319383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786331702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590,"session_id":"5cb363e7-2a80-41fb-bd7c-c6c0a5e9983d"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:53.786344292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 222 len 24576 data: 7 5 4 3 3 4 02/30 Read block 278 len 36864 03/30 Read block 251 len 8192 04/30 Write block 16 len 32768 data: 8 10 9 11 12 11 12 9 05/30 Read block 17 len 8192 06/30 Read block 450 len 8192 07/30 Read block 218 len 12288 08/30 Write block 93 len 4096 data: 11 09/30 Write block 578 len 4096 data: 7 10/30 Read block 325 len 28672 11/30 Read block 80 len 28672 12/30 Flush 13/30 Write block 120 len 20480 data: 13 11 10 10 10 14/30 Write block 304 len 28672 data: 6 8 7 8 9 7 8 15/30 Flush 16/30 Write block 391 len 36864 data: 9 11 10 12 14 14 14 14 13 17/30 Read block 302 len 16384 18/30 Read block 345 len 24576 19/30 Write block 255 len 12288 data: 13 13 10 20/30 Read block 588 len 12288 21/30 Read block 278 len 24576 22/30 Read block 582 len 36864 23/30 Read block 215 len 20480 24/30 Read block 599 len 4096 25/30 Read block 135 len 36864 26/30 Read block 275 len 36864 27/30 Read block 189 len 16384 28/30 Read block 424 len 32768 29/30 Write block 125 len 32768 data: 11 14 15 16 13 13 13 13 30/30 Write block 213 len 36864 data: 7 5 7 11 11 9 9 8 7 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 9 Done Done Done false 1016 Acked Read 4 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Read 3 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 5 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 13 13 13 39 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:53.810049966Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1590} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 184 184 184 F F F 1 020-039 102 102 102 184 184 184 F F F 2 040-059 101 101 101 183 183 183 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 102 102 102 184 184 184 F F F 5 100-119 100 100 100 182 182 182 F F F 6 120-139 102 102 102 185 185 185 T T T 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 91 91 91 165 165 165 T T T 11 220-239 102 102 102 184 184 184 T T T 12 240-259 96 96 96 176 176 176 T T T 13 260-279 100 100 100 181 181 181 F F F 14 280-299 100 100 100 181 181 181 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 101 101 101 183 183 183 F F F 17 340-359 101 101 101 183 183 183 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 96 96 96 174 174 174 T T T 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 99 99 99 179 179 179 F F F 26 520-539 100 100 100 181 181 181 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 102 102 102 184 184 184 F F F 29 580-599 100 100 100 180 180 180 F F F Max gen: 102, Max flush: 185 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 102 102 102 184 184 184 F F F 1 020-039 102 102 102 184 184 184 F F F 2 040-059 101 101 101 183 183 183 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 102 102 102 184 184 184 F F F 5 100-119 100 100 100 182 182 182 F F F 6 120-139 103 103 103 186 186 186 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 96 96 96 173 173 173 F F F 10 200-219 103 103 103 186 186 186 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 103 103 103 186 186 186 F F F 13 260-279 100 100 100 181 181 181 F F F 14 280-299 100 100 100 181 181 181 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 101 101 101 183 183 183 F F F 17 340-359 101 101 101 183 183 183 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 103 103 103 186 186 186 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 95 95 95 171 171 171 F F F 24 480-499 95 95 95 171 171 171 F F F 25 500-519 99 99 99 179 179 179 F F F 26 520-539 100 100 100 181 181 181 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 102 102 102 184 184 184 F F F 29 580-599 100 100 100 180 180 180 F F F Max gen: 103, Max flush: 186 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.146774946Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.190245867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.190918812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.190957741Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.19097287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.19098772Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} {"msg":"Crucible 7da79840-e641-4a96-901f-c241803d9f67 has session id: 40a791cf-5ab0-48af-8498-427984629800","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.191443944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Upstairs opts: Upstairs UUID: 7da79840-e641-4a96-901f-c241803d9f67, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.191488622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Crucible stats registered with UUID: 7da79840-e641-4a96-901f-c241803d9f67","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.191518721Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221270727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221304295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} {"msg":"7da79840-e641-4a96-901f-c241803d9f67 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221339084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221356324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221385612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221403882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221421171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.22144873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221482829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221514828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221549497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221585715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_connection connected from Ok(127.0.0.1:35574)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221622384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"io task","client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_connection connected from Ok(127.0.0.1:34323)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221654733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"io task","client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_connection connected from Ok(127.0.0.1:55465)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.221799048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"io task","client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.222450494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.222490883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.222521542Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.22255407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.222585119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.222612978Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223087741Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223165688Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.2233891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223627142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223653451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.22366793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223695949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223710299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223723988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223848234Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223871303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[0]R flush_numbers[0..12]: [184, 184, 183, 183, 184, 182, 186, 179, 177, 173, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223886532Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[0]R generation[0..12]: [102, 102, 101, 101, 102, 100, 103, 99, 97, 96, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223900652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223914581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[1]R flush_numbers[0..12]: [184, 184, 183, 183, 184, 182, 186, 179, 177, 173, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223929191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[1]R generation[0..12]: [102, 102, 101, 101, 102, 100, 103, 99, 97, 96, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.22394301Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.2239572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[2]R flush_numbers[0..12]: [184, 184, 183, 183, 184, 182, 186, 179, 177, 173, 186, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223971229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[2]R generation[0..12]: [102, 102, 101, 101, 102, 100, 103, 99, 97, 96, 103, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.223984909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224003978Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224027447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224042177Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Next flush: 187","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:06:58.224055286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224078485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"":"downstairs","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224092315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224104414Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"0","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224117664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"1","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224130864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"client":"2","":"downstairs","session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224144003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224156203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"7da79840-e641-4a96-901f-c241803d9f67 is now active with session: 40a791cf-5ab0-48af-8498-427984629800","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224169052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224181252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:06:58.224193561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 590 len 40960 data: 10 6 6 7 7 5 5 5 5 4 02/30 Read block 442 len 12288 03/30 Write block 469 len 24576 data: 8 8 8 7 10 11 04/30 Write block 567 len 12288 data: 12 10 11 05/30 Flush 06/30 Write block 193 len 32768 data: 8 9 6 7 8 4 4 4 07/30 Flush 08/30 Write block 120 len 36864 data: 14 12 11 11 11 12 15 16 17 09/30 Write block 7 len 8192 data: 4 3 10/30 Read block 270 len 28672 11/30 Write block 30 len 12288 data: 12 13 10 12/30 Read block 472 len 8192 13/30 Write block 474 len 40960 data: 12 10 7 6 4 6 6 6 6 7 14/30 Read block 16 len 12288 15/30 Read block 348 len 36864 16/30 Flush 17/30 Write block 383 len 20480 data: 10 11 12 11 8 18/30 Read block 159 len 24576 19/30 Write block 259 len 16384 data: 5 5 3 3 20/30 Read block 386 len 36864 21/30 Read block 3 len 40960 22/30 Write block 493 len 40960 data: 8 11 9 11 11 9 11 10 9 10 23/30 Flush 24/30 Write block 203 len 16384 data: 9 11 11 11 25/30 Write block 275 len 32768 data: 11 10 11 11 9 11 12 14 26/30 Write block 55 len 16384 data: 7 7 7 9 27/30 Write block 381 len 20480 data: 12 11 11 12 13 28/30 Read block 97 len 12288 29/30 Write block 342 len 8192 data: 11 11 30/30 Read block 117 len 8192 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 5 Sent Sent Done false 1027 Acked Read 3 Sent Sent Done false 1028 Acked Write 2 Sent Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 5 0 9 Done 3 2 7 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.251352331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598} Wrote out file "/tmp/test_repair_verify.data" {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.251997818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252037586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252063465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252089284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252115113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252138753Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252159152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252183181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25220576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252228679Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252251798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252301187Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252326396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252351275Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252373594Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252393353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252412272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252433372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252455311Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25247079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.2524835Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25249592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252508349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252520649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252533028Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252545328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252557487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252569587Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252581777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252594146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252608636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252621005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252633325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252645484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252657994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252670133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252682363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252694502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252706632Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252718941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252731201Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252743411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25275573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25276812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252780349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252792519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252804738Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252818528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252831037Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252843377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252855656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252867876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252880546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252892845Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252905195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252917424Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252929664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252941813Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252953863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252965953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252978182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.252990452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253002811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253015141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25302892Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25304126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25305346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253065609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253077719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253089928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253102188Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253114507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253126717Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253138756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253151006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253163245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253175345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253187525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253199754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253211754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253223903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253237503Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253249922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253262132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253274232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253286351Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253298341Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25331041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25332269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253334859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253347349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253359598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253372068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253384297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253396637Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253409027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253421426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253433846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253446325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253460285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253472994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253485484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253497843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253510463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253522932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253535352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253547692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253560161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253572611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25358488Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.25359711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253609179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253621449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253633528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253645718Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253657948Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253671757Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 2; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253684257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253696466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 0; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253708766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253720945Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} {"msg":"IO channel closed for 1; we are hopefully exiting","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:06:58.253733195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1598,"session_id":"40a791cf-5ab0-48af-8498-427984629800"} Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 104 104 104 189 189 189 F F F 2 040-059 101 101 101 183 183 183 T T T 3 060-079 101 101 101 183 183 183 F F F 4 080-099 102 102 102 184 184 184 F F F 5 100-119 100 100 100 182 182 182 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 104 104 104 188 188 188 T T T 11 220-239 103 103 103 186 186 186 F F F 12 240-259 104 104 104 190 190 190 F F F 13 260-279 104 104 104 190 190 190 T T T 14 280-299 100 100 100 181 181 181 T T T 15 300-319 102 102 102 185 185 185 F F F 16 320-339 101 101 101 183 183 183 F F F 17 340-359 101 101 101 183 183 183 T T T 18 360-379 96 96 96 174 174 174 F F F 19 380-399 104 104 104 190 190 190 T T T 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 100 100 100 181 181 181 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 104 104 104 187 187 187 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 104, Max flush: 190 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 104 104 104 189 189 189 F F F 2 040-059 105 105 105 191 191 191 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 102 102 102 184 184 184 F F F 5 100-119 100 100 100 182 182 182 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 99 99 99 179 179 179 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 105 105 105 191 191 191 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 104 104 104 190 190 190 F F F 13 260-279 105 105 105 191 191 191 F F F 14 280-299 105 105 105 191 191 191 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 101 101 101 183 183 183 F F F 17 340-359 105 105 105 191 191 191 F F F 18 360-379 96 96 96 174 174 174 F F F 19 380-399 105 105 105 191 191 191 F F F 20 400-419 97 97 97 177 177 177 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 100 100 100 181 181 181 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 104 104 104 187 187 187 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 105, Max flush: 191 Begin loop 53 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.5783989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.622614794Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625369825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625428702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625449691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625468991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} {"msg":"Crucible 3009775f-ee01-4222-938f-07c8e75a6b38 has session id: 05955eff-93e9-49c9-ab59-b673e93434d5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625901595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Upstairs opts: Upstairs UUID: 3009775f-ee01-4222-938f-07c8e75a6b38, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625939514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Crucible stats registered with UUID: 3009775f-ee01-4222-938f-07c8e75a6b38","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.625956853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656166163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656199261Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} {"msg":"3009775f-ee01-4222-938f-07c8e75a6b38 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656270279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656292958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656321757Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656339727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656358146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656374865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656391625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656408194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656425513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656442523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:63590)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656582148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"io task","client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59286)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656627296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"io task","client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_connection connected from Ok(127.0.0.1:33787)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.656659255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"io task","client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657338971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657387899Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657434447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657465806Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657490735Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657506324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.657980937Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658121052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658151441Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.65847189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658499349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658515038Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658751999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658791438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658817657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658840336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658865405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[0]R flush_numbers[0..12]: [189, 189, 191, 183, 184, 182, 189, 179, 177, 188, 191, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658897944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[0]R generation[0..12]: [104, 104, 105, 101, 102, 100, 104, 99, 97, 104, 105, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658924073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658950282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[1]R flush_numbers[0..12]: [189, 189, 191, 183, 184, 182, 189, 179, 177, 188, 191, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.658977691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[1]R generation[0..12]: [104, 104, 105, 101, 102, 100, 104, 99, 97, 104, 105, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.65900536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659030949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[2]R flush_numbers[0..12]: [189, 189, 191, 183, 184, 182, 189, 179, 177, 188, 191, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659059608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[2]R generation[0..12]: [104, 104, 105, 101, 102, 100, 104, 99, 97, 104, 105, 103]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:07:02.659090147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659127026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659156755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659187294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Next flush: 192","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659216703Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659247162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"":"downstairs","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.65928107Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 } {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659313909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"0","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659338748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"1","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659353178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"client":"2","":"downstairs","session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659366707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659379377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"3009775f-ee01-4222-938f-07c8e75a6b38 is now active with session: 05955eff-93e9-49c9-ab59-b673e93434d5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659392436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659404536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606,"session_id":"05955eff-93e9-49c9-ab59-b673e93434d5"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:02.659416885Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 25 len 16384 data: 14 13 12 13 02/30 Write block 296 len 16384 data: 8 9 11 9 03/30 Write block 265 len 12288 data: 6 7 9 04/30 Read block 7 len 20480 05/30 Read block 149 len 32768 06/30 Write block 50 len 28672 data: 9 7 9 10 11 8 8 07/30 Read block 99 len 8192 08/30 Write block 110 len 8192 data: 7 8 09/30 Flush 10/30 Write block 377 len 36864 data: 11 10 12 12 13 12 12 13 14 11/30 Read block 19 len 12288 12/30 Write block 206 len 12288 data: 12 9 7 13/30 Flush 14/30 Write block 412 len 24576 data: 8 8 7 5 6 5 15/30 Write block 29 len 8192 data: 12 13 16/30 Flush 17/30 Write block 279 len 4096 data: 10 18/30 Read block 176 len 28672 19/30 Flush 20/30 Write block 330 len 8192 data: 9 11 21/30 Read block 0 len 20480 22/30 Read block 116 len 40960 23/30 Flush 24/30 Write block 106 len 28672 data: 11 8 9 7 8 9 10 25/30 Write block 532 len 32768 data: 9 8 8 7 7 7 8 5 26/30 Read block 560 len 20480 27/30 Read block 57 len 32768 28/30 Write block 145 len 16384 data: 9 10 11 8 29/30 Write block 208 len 20480 data: 8 10 8 7 5 30/30 Read block 360 len 20480 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 5 Sent Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 2 4 Done 5 7 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:02.685553322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1606} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 106 106 106 194 194 194 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 102 102 102 184 184 184 F F F 5 100-119 106 106 106 192 192 192 T T T 6 120-139 104 104 104 189 189 189 F F F 7 140-159 99 99 99 179 179 179 T T T 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 106 106 106 193 193 193 T T T 11 220-239 103 103 103 186 186 186 F F F 12 240-259 104 104 104 190 190 190 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 106 106 106 196 196 196 F F F 17 340-359 105 105 105 191 191 191 F F F 18 360-379 106 106 106 193 193 193 F F F 19 380-399 106 106 106 193 193 193 F F F 20 400-419 106 106 106 194 194 194 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 100 100 100 181 181 181 T T T 27 540-559 100 100 100 180 180 180 F F F 28 560-579 104 104 104 187 187 187 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 106, Max flush: 196 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 106 106 106 194 194 194 F F F 2 040-059 106 106 106 192 192 192 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 102 102 102 184 184 184 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 107 107 107 197 197 197 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 107 107 107 197 197 197 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 104 104 104 190 190 190 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 106 106 106 196 196 196 F F F 17 340-359 105 105 105 191 191 191 F F F 18 360-379 106 106 106 193 193 193 F F F 19 380-399 106 106 106 193 193 193 F F F 20 400-419 106 106 106 194 194 194 F F F 21 420-439 94 94 94 168 168 168 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 104 104 104 190 190 190 F F F 26 520-539 107 107 107 197 197 197 F F F 27 540-559 100 100 100 180 180 180 F F F 28 560-579 104 104 104 187 187 187 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 107, Max flush: 197 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:06.98489623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.028528304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.029105924Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.029144742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.029172891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.02918952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} {"msg":"Crucible d74cfef9-5445-442d-998a-7ec5c49a6ed4 has session id: 786b32c9-1bf3-4721-a288-a84120e6665d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.029677153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Upstairs opts: Upstairs UUID: d74cfef9-5445-442d-998a-7ec5c49a6ed4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.029718852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Crucible stats registered with UUID: d74cfef9-5445-442d-998a-7ec5c49a6ed4","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.029743511Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059483477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059519276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} {"msg":"d74cfef9-5445-442d-998a-7ec5c49a6ed4 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059554995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059573474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059604123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059636061Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.05967189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059708879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059745588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059779256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059815625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059850914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_connection connected from Ok(127.0.0.1:39668)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.059967709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"io task","client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_connection connected from Ok(127.0.0.1:37407)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060017108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"io task","client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_connection connected from Ok(127.0.0.1:44432)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060055636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"io task","client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.06050723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060542809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060560498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060642385Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060835568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.060867037Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.061399338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.061429407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.061445246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.061943238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.061972277Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.061992707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062060594Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062089623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062105442Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062118282Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062132202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[0]R flush_numbers[0..12]: [189, 194, 192, 183, 184, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062146291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[0]R generation[0..12]: [104, 106, 106, 101, 102, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.06216035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.06217458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[1]R flush_numbers[0..12]: [189, 194, 192, 183, 184, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062191819Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[1]R generation[0..12]: [104, 106, 106, 101, 102, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062206269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062220788Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[2]R flush_numbers[0..12]: [189, 194, 192, 183, 184, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062234598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[2]R generation[0..12]: [104, 106, 106, 101, 102, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062248247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062262047Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062275146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062288756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062301815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062314855Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"":"downstairs","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:07:07.062328484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062408212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"0","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062427491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"1","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.06244133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"client":"2","":"downstairs","session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.06245743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062470169Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"d74cfef9-5445-442d-998a-7ec5c49a6ed4 is now active with session: 786b32c9-1bf3-4721-a288-a84120e6665d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062483149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062495478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614,"session_id":"786b32c9-1bf3-4721-a288-a84120e6665d"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:07.062507968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 504 len 4096 data: 11 02/30 Write block 321 len 36864 data: 9 9 7 5 5 5 6 7 7 03/30 Write block 248 len 20480 data: 10 11 10 14 14 04/30 Read block 45 len 20480 05/30 Write block 45 len 16384 data: 8 8 8 7 06/30 Read block 588 len 8192 07/30 Read block 2 len 32768 08/30 Read block 25 len 4096 09/30 Write block 348 len 12288 data: 10 10 5 10/30 Read block 70 len 36864 11/30 Read block 6 len 20480 12/30 Write block 22 len 4096 data: 13 13/30 Read block 557 len 16384 14/30 Write block 547 len 8192 data: 12 15 15/30 Read block 423 len 20480 16/30 Write block 519 len 24576 data: 8 9 10 9 9 9 17/30 Read block 98 len 12288 18/30 Write block 401 len 32768 data: 8 6 5 4 4 4 5 6 19/30 Write block 87 len 24576 data: 11 12 11 13 9 10 20/30 Read block 538 len 4096 21/30 Read block 242 len 16384 22/30 Write block 429 len 28672 data: 6 5 5 7 7 7 6 23/30 Read block 285 len 8192 24/30 Flush 25/30 Write block 25 len 28672 data: 15 14 13 14 13 14 14 26/30 Write block 521 len 16384 data: 11 10 10 10 27/30 Write block 341 len 4096 data: 9 28/30 Read block 536 len 12288 29/30 Read block 447 len 28672 30/30 Read block 176 len 24576 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 7 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 7 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:07.084910859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1614} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 108 108 108 198 198 198 T T T 2 040-059 108 108 108 198 198 198 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 108 108 108 198 198 198 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 107 107 107 197 197 197 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 107 107 107 197 197 197 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 108 108 108 198 198 198 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 108 108 108 198 198 198 T T T 18 360-379 106 106 106 193 193 193 F F F 19 380-399 106 106 106 193 193 193 F F F 20 400-419 108 108 108 198 198 198 F F F 21 420-439 108 108 108 198 198 198 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 108 108 108 198 198 198 T T T 27 540-559 108 108 108 198 198 198 F F F 28 560-579 104 104 104 187 187 187 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 108, Max flush: 198 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 108 108 108 198 198 198 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 108 108 108 198 198 198 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 107 107 107 197 197 197 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 107 107 107 197 197 197 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 108 108 108 198 198 198 F F F 13 260-279 106 106 106 195 195 195 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 108 108 108 198 198 198 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 106 106 106 193 193 193 F F F 19 380-399 106 106 106 193 193 193 F F F 20 400-419 108 108 108 198 198 198 F F F 21 420-439 108 108 108 198 198 198 F F F 22 440-459 96 96 96 173 173 173 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 108 108 108 198 198 198 F F F 26 520-539 109 109 109 199 199 199 F F F 27 540-559 108 108 108 198 198 198 F F F 28 560-579 104 104 104 187 187 187 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 109, Max flush: 199 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.37283357Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:11.414748006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.415146422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.446092875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.446662305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.446699073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.446713643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.446728322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} {"msg":"Crucible 560acab2-e064-4f98-a9d4-9c0079d4df2c has session id: d9aea9e0-2908-491b-a60b-6c71b21481f2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.447243984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Upstairs opts: Upstairs UUID: 560acab2-e064-4f98-a9d4-9c0079d4df2c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.447273272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Crucible stats registered with UUID: 560acab2-e064-4f98-a9d4-9c0079d4df2c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.447288002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477461683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477492001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} {"msg":"560acab2-e064-4f98-a9d4-9c0079d4df2c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477558689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477578058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477600668Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477616017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477633807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477658195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477716584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477738983Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477757882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477775421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60768)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477872278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"io task","client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58709)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477912926Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"io task","client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_connection connected from Ok(127.0.0.1:61207)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.477941715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"io task","client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.478664739Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.478706998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.478738927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.478771086Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.478799754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.478821833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479242808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479327565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479394833Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479850287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479878495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479893025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479905645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479918954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479932273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479944613Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479958003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[0]R flush_numbers[0..12]: [189, 199, 198, 183, 198, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479975942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[0]R generation[0..12]: [104, 109, 108, 101, 108, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.479990031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480004261Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[1]R flush_numbers[0..12]: [189, 199, 198, 183, 198, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.48001827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[1]R generation[0..12]: [104, 109, 108, 101, 108, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.48003582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480049999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[2]R flush_numbers[0..12]: [189, 199, 198, 183, 198, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480063829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[2]R generation[0..12]: [104, 109, 108, 101, 108, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480077398Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480091238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480104047Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480117267Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Next flush: 200","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480130386Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480143606Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480164755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480176865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"0","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480190544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"1","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480204004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"client":"2","":"downstairs","session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480217293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480229733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"560acab2-e064-4f98-a9d4-9c0079d4df2c is now active with session: d9aea9e0-2908-491b-a60b-6c71b21481f2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480244892Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480257322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622,"session_id":"d9aea9e0-2908-491b-a60b-6c71b21481f2"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:11.480269621Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 557 len 16384 data: 6 4 4 6 02/30 Read block 133 len 24576 03/30 Read block 532 len 4096 04/30 Flush 05/30 Write block 330 len 36864 data: 10 12 13 14 13 14 14 13 14 06/30 Read block 286 len 16384 07/30 Read block 141 len 28672 08/30 Read block 494 len 8192 09/30 Read block 482 len 8192 10/30 Read block 275 len 24576 11/30 Write block 444 len 4096 data: 3 12/30 Write block 445 len 12288 data: 4 6 8 13/30 Write block 502 len 12288 data: 11 11 12 14/30 Read block 267 len 8192 15/30 Write block 447 len 36864 data: 9 7 8 8 6 7 11 9 9 16/30 Read block 315 len 20480 17/30 Flush 18/30 Write block 553 len 24576 data: 16 10 9 7 7 5 19/30 Read block 77 len 24576 20/30 Read block 246 len 4096 21/30 Write block 378 len 4096 data: 11 22/30 Write block 446 len 4096 data: 7 23/30 Read block 204 len 40960 24/30 Read block 377 len 40960 25/30 Write block 257 len 40960 data: 11 7 6 6 4 4 3 3 7 8 26/30 Read block 399 len 36864 27/30 Read block 387 len 28672 28/30 Write block 527 len 20480 data: 6 5 5 7 10 29/30 Read block 395 len 8192 30/30 Read block 504 len 8192 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 6 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 13 12 12 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:11.50301676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1622} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 108 108 108 198 198 198 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 108 108 108 198 198 198 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 107 107 107 197 197 197 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 107 107 107 197 197 197 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 108 108 108 198 198 198 T T T 13 260-279 106 106 106 195 195 195 T T T 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 110 110 110 201 201 201 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 106 106 106 193 193 193 T T T 19 380-399 106 106 106 193 193 193 F F F 20 400-419 108 108 108 198 198 198 F F F 21 420-439 108 108 108 198 198 198 F F F 22 440-459 110 110 110 201 201 201 T T T 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 110 110 110 201 201 201 F F F 26 520-539 109 109 109 199 199 199 T T T 27 540-559 110 110 110 200 200 200 T T T 28 560-579 110 110 110 200 200 200 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 110, Max flush: 201 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 108 108 108 198 198 198 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 108 108 108 198 198 198 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 104 104 104 189 189 189 F F F 7 140-159 107 107 107 197 197 197 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 F F F 10 200-219 107 107 107 197 197 197 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 111 111 111 202 202 202 F F F 13 260-279 111 111 111 202 202 202 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 110 110 110 201 201 201 F F F 17 340-359 109 109 109 199 199 199 F F F 18 360-379 111 111 111 202 202 202 F F F 19 380-399 106 106 106 193 193 193 F F F 20 400-419 108 108 108 198 198 198 F F F 21 420-439 108 108 108 198 198 198 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 110 110 110 201 201 201 F F F 26 520-539 111 111 111 202 202 202 F F F 27 540-559 111 111 111 202 202 202 F F F 28 560-579 110 110 110 200 200 200 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 111, Max flush: 202 Begin loop 56 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.855444292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.898676381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899317928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899356076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899374916Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899389985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} {"msg":"Crucible 5cb8bf97-a935-40f0-8f3f-ea942b0a6537 has session id: 2362432c-d7bc-42cd-9a2b-74745d3164ce","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899855058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Upstairs opts: Upstairs UUID: 5cb8bf97-a935-40f0-8f3f-ea942b0a6537, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899896887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Crucible stats registered with UUID: 5cb8bf97-a935-40f0-8f3f-ea942b0a6537","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.899920676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929634214Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929666232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} {"msg":"5cb8bf97-a935-40f0-8f3f-ea942b0a6537 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929715231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.92973472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929761619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929779408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929797038Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929813617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929830247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929862975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929893864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929928993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:34427)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929959372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"io task","client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:42437)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.929978551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"io task","client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_connection connected from Ok(127.0.0.1:34472)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.930027729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"io task","client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.930789262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.930816781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.93083184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.93084598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.930961256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.930982335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.931551704Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.931582653Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.931631851Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932120574Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932145043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932159432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932239509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932258359Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932272008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932284778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932298407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[0]R flush_numbers[0..12]: [189, 199, 198, 183, 198, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932312587Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[0]R generation[0..12]: [104, 109, 108, 101, 108, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932326396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932340356Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[1]R flush_numbers[0..12]: [189, 199, 198, 183, 198, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932354425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[1]R generation[0..12]: [104, 109, 108, 101, 108, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932368305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932397774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[2]R flush_numbers[0..12]: [189, 199, 198, 183, 198, 197, 189, 197, 177, 188, 197, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932416783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[2]R generation[0..12]: [104, 109, 108, 101, 108, 107, 104, 107, 97, 104, 107, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932431182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932445192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932458272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932471621Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Next flush: 203","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2024-11-07T14:07:15.932484721Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.93250555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"":"downstairs","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932518899Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932530989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"0","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932544229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"1","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932557318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"client":"2","":"downstairs","session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932570608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932583007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"5cb8bf97-a935-40f0-8f3f-ea942b0a6537 is now active with session: 2362432c-d7bc-42cd-9a2b-74745d3164ce","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932595787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932607876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630,"session_id":"2362432c-d7bc-42cd-9a2b-74745d3164ce"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:15.932620196Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 132 len 4096 data: 14 02/30 Write block 87 len 12288 data: 12 13 12 03/30 Read block 430 len 8192 04/30 Read block 574 len 28672 05/30 Read block 31 len 20480 06/30 Read block 425 len 8192 07/30 Read block 557 len 24576 08/30 Read block 391 len 32768 09/30 Write block 529 len 4096 data: 6 10/30 Read block 557 len 16384 11/30 Write block 145 len 12288 data: 10 11 12 12/30 Read block 232 len 4096 13/30 Read block 191 len 16384 14/30 Write block 364 len 32768 data: 11 8 8 8 8 12 14 15 15/30 Read block 330 len 12288 16/30 Read block 62 len 4096 17/30 Write block 186 len 16384 data: 11 10 8 9 18/30 Read block 170 len 32768 19/30 Read block 93 len 16384 20/30 Write block 261 len 16384 data: 5 5 4 4 21/30 Read block 43 len 4096 22/30 Read block 25 len 4096 23/30 Read block 31 len 28672 24/30 Read block 55 len 12288 25/30 Read block 227 len 24576 26/30 Write block 183 len 24576 data: 9 8 10 12 11 9 27/30 Write block 415 len 20480 data: 6 7 6 7 7 28/30 Write block 342 len 12288 data: 12 12 11 29/30 Read block 264 len 24576 30/30 Write block 194 len 40960 data: 10 7 8 9 5 5 5 5 8 10 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 2 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 8 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 1 Done Done Done false 1012 Acked Read 4 Done Done Done false 1013 Acked Write 8 Done Done Done false 1014 Acked Read 3 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 4 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 1 Done Done Done false 1021 Acked Read 1 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 3 5 Done 29 29 27 85 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:15.950925345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1630} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 108 108 108 198 198 198 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 108 108 108 198 198 198 T T T 5 100-119 107 107 107 197 197 197 F F F 6 120-139 104 104 104 189 189 189 T T T 7 140-159 107 107 107 197 197 197 T T T 8 160-179 97 97 97 177 177 177 F F F 9 180-199 104 104 104 188 188 188 T T T 10 200-219 107 107 107 197 197 197 T T T 11 220-239 103 103 103 186 186 186 F F F 12 240-259 111 111 111 202 202 202 F F F 13 260-279 111 111 111 202 202 202 T T T 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 110 110 110 201 201 201 F F F 17 340-359 109 109 109 199 199 199 T T T 18 360-379 111 111 111 202 202 202 T T T 19 380-399 106 106 106 193 193 193 F F F 20 400-419 108 108 108 198 198 198 T T T 21 420-439 108 108 108 198 198 198 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 110 110 110 201 201 201 F F F 26 520-539 111 111 111 202 202 202 T T T 27 540-559 111 111 111 202 202 202 F F F 28 560-579 110 110 110 200 200 200 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 111, Max flush: 202 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 F F F 1 020-039 109 109 109 199 199 199 F F F 2 040-059 108 108 108 198 198 198 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 112 112 112 203 203 203 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 112 112 112 203 203 203 F F F 8 160-179 97 97 97 177 177 177 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 103 103 103 186 186 186 F F F 12 240-259 111 111 111 202 202 202 F F F 13 260-279 112 112 112 203 203 203 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 110 110 110 201 201 201 F F F 17 340-359 112 112 112 203 203 203 F F F 18 360-379 112 112 112 203 203 203 F F F 19 380-399 106 106 106 193 193 193 F F F 20 400-419 112 112 112 203 203 203 F F F 21 420-439 108 108 108 198 198 198 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 104 104 104 189 189 189 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 110 110 110 201 201 201 F F F 26 520-539 112 112 112 203 203 203 F F F 27 540-559 111 111 111 202 202 202 F F F 28 560-579 110 110 110 200 200 200 F F F 29 580-599 104 104 104 187 187 187 F F F Max gen: 112, Max flush: 203 Begin loop 57 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.337165956Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:20.379055494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.37944867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.410114773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.410670973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.410727651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.4107483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.410766539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} {"msg":"Crucible 38ea20fa-7665-4110-8404-2118c323d24d has session id: a0b73085-fcea-4cef-9116-44b38abdc315","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.411225123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Upstairs opts: Upstairs UUID: 38ea20fa-7665-4110-8404-2118c323d24d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.411268421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Crucible stats registered with UUID: 38ea20fa-7665-4110-8404-2118c323d24d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.411293491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440783136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440815105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} {"msg":"38ea20fa-7665-4110-8404-2118c323d24d active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440845154Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440862103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440888262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440920661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440960829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.440997308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441031867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441067696Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441104664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441292278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_connection connected from Ok(127.0.0.1:35903)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441333016Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"io task","client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_connection connected from Ok(127.0.0.1:45758)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441359615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"io task","client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_connection connected from Ok(127.0.0.1:50340)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441386074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"io task","client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441807719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441848837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441881166Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441913745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441943984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.441972483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.442411957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.442452916Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.442522473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443030115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443059204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443073723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443086913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443100912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443114322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443126801Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443140421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[0]R flush_numbers[0..12]: [189, 199, 198, 183, 203, 197, 203, 203, 177, 203, 203, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.44315465Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[0]R generation[0..12]: [104, 109, 108, 101, 112, 107, 112, 112, 97, 112, 112, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.44316896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443183269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[1]R flush_numbers[0..12]: [189, 199, 198, 183, 203, 197, 203, 203, 177, 203, 203, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443197279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[1]R generation[0..12]: [104, 109, 108, 101, 112, 107, 112, 112, 97, 112, 112, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443216148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443230738Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[2]R flush_numbers[0..12]: [189, 199, 198, 183, 203, 197, 203, 203, 177, 203, 203, 186]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443244577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[2]R generation[0..12]: [104, 109, 108, 101, 112, 107, 112, 112, 97, 112, 112, 103]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443258427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443272506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Max found gen is 113","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443285766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Generation requested: 114 >= found:113","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443298965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} Wait for a query_work_queue command to finish before sending IO {"msg":"Next flush: 204","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443321844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443335094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"":"downstairs","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443348513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443360783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"0","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443374073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"1","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443387372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"client":"2","":"downstairs","session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443400661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443413181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"38ea20fa-7665-4110-8404-2118c323d24d is now active with session: a0b73085-fcea-4cef-9116-44b38abdc315","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.443428191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.4434405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638,"session_id":"a0b73085-fcea-4cef-9116-44b38abdc315"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:20.44345298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 354 len 32768 data: 6 8 8 8 9 10 12 11 02/30 Write block 179 len 4096 data: 7 03/30 Write block 49 len 4096 data: 6 04/30 Read block 593 len 16384 05/30 Write block 339 len 32768 data: 9 7 10 13 13 12 9 8 06/30 Flush 07/30 Write block 345 len 24576 data: 10 9 9 11 11 6 08/30 Write block 221 len 32768 data: 8 8 6 5 4 4 5 4 09/30 Write block 514 len 4096 data: 9 10/30 Read block 359 len 4096 11/30 Write block 337 len 20480 data: 14 15 10 8 11 12/30 Write block 14 len 28672 data: 8 7 9 11 10 12 13 13/30 Write block 381 len 12288 data: 14 13 13 14/30 Write block 474 len 8192 data: 13 11 15/30 Read block 203 len 32768 16/30 Read block 140 len 36864 17/30 Write block 409 len 28672 data: 9 10 9 9 9 8 7 18/30 Write block 534 len 36864 data: 9 8 8 8 9 6 6 6 5 19/30 Read block 366 len 12288 20/30 Write block 590 len 40960 data: 11 7 7 8 8 6 6 6 6 5 21/30 Read block 0 len 28672 22/30 Read block 139 len 8192 23/30 Read block 376 len 24576 24/30 Read block 72 len 12288 25/30 Read block 588 len 28672 26/30 Read block 229 len 36864 27/30 Write block 428 len 36864 data: 6 7 6 6 8 8 8 7 7 28/30 Write block 250 len 40960 data: 11 15 15 13 10 14 14 12 8 7 29/30 Read block 583 len 40960 30/30 Read block 491 len 28672 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Write 8 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Write 3 Done Done Done false 1013 Acked Write 2 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Read 9 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Write 9 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 10 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 9 Done Done Done false 1026 Acked Write 9 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Read 7 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 24 24 23 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1005 1005 1005 Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:20.464938054Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1638} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 104 104 104 189 189 189 T T T 1 020-039 109 109 109 199 199 199 T T T 2 040-059 114 114 114 204 204 204 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 112 112 112 203 203 203 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 112 112 112 203 203 203 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 103 103 103 186 186 186 T T T 12 240-259 111 111 111 202 202 202 T T T 13 260-279 112 112 112 203 203 203 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 114 114 114 204 204 204 T T T 17 340-359 114 114 114 204 204 204 T T T 18 360-379 114 114 114 204 204 204 F F F 19 380-399 106 106 106 193 193 193 T T T 20 400-419 112 112 112 203 203 203 T T T 21 420-439 108 108 108 198 198 198 T T T 22 440-459 111 111 111 202 202 202 F F F 23 460-479 104 104 104 189 189 189 T T T 24 480-499 104 104 104 190 190 190 F F F 25 500-519 110 110 110 201 201 201 T T T 26 520-539 112 112 112 203 203 203 T T T 27 540-559 111 111 111 202 202 202 T T T 28 560-579 110 110 110 200 200 200 F F F 29 580-599 104 104 104 187 187 187 T T T Max gen: 114, Max flush: 204 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 114 114 114 204 204 204 F F F 3 060-079 101 101 101 183 183 183 F F F 4 080-099 112 112 112 203 203 203 F F F 5 100-119 107 107 107 197 197 197 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 112 112 112 203 203 203 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 112 112 112 203 203 203 F F F 11 220-239 115 115 115 205 205 205 F F F 12 240-259 115 115 115 205 205 205 F F F 13 260-279 112 112 112 203 203 203 F F F 14 280-299 106 106 106 192 192 192 F F F 15 300-319 102 102 102 185 185 185 F F F 16 320-339 115 115 115 205 205 205 F F F 17 340-359 115 115 115 205 205 205 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 115 115 115 205 205 205 F F F 20 400-419 115 115 115 205 205 205 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 104 104 104 190 190 190 F F F 25 500-519 115 115 115 205 205 205 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 115 115 115 205 205 205 F F F 28 560-579 110 110 110 200 200 200 F F F 29 580-599 115 115 115 205 205 205 F F F Max gen: 115, Max flush: 205 Begin loop 58 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.929422211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.972514505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973110024Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973152412Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973170202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973188171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} {"msg":"Crucible dcdba768-0624-456c-b771-1cfd06ff6613 has session id: d00c483a-a4b0-4f1d-918e-cf00acdcc92e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973694123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Upstairs opts: Upstairs UUID: dcdba768-0624-456c-b771-1cfd06ff6613, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973725672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Crucible stats registered with UUID: dcdba768-0624-456c-b771-1cfd06ff6613","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:24.973741391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.004834859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.004868988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} {"msg":"dcdba768-0624-456c-b771-1cfd06ff6613 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.004918816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.004938485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.004965594Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.004988163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005020392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005055991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.00508054Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005097759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005115408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005134558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60178)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005161027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"io task","client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_connection connected from Ok(127.0.0.1:50561)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005189936Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"io task","client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57236)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.00534152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"io task","client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005831353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.005873071Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006003507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006031775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006253527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006299876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006402062Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006540737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.006980331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007141495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007166445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007181384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007194213Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007207693Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007221112Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007489063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007516312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 204, 183, 203, 197, 203, 203, 204, 203, 203, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007532051Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[0]R generation[0..12]: [115, 115, 114, 101, 112, 107, 112, 112, 114, 112, 112, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007546921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.00756141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 204, 183, 203, 197, 203, 203, 204, 203, 203, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.0075803Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[1]R generation[0..12]: [115, 115, 114, 101, 112, 107, 112, 112, 114, 112, 112, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007594909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007622618Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 204, 183, 203, 197, 203, 203, 204, 203, 203, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007636897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[2]R generation[0..12]: [115, 115, 114, 101, 112, 107, 112, 112, 114, 112, 112, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007650747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007664767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007677986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007691435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Next flush: 206","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007704745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007718045Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"":"downstairs","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007742184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007754893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"0","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007768433Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"1","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007782002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"client":"2","":"downstairs","session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007795472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007810421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"dcdba768-0624-456c-b771-1cfd06ff6613 is now active with session: d00c483a-a4b0-4f1d-918e-cf00acdcc92e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.007823651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.00783595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646,"session_id":"d00c483a-a4b0-4f1d-918e-cf00acdcc92e"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:25.00784868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 490 len 32768 data: 6 7 9 9 12 10 12 12 02/30 Write block 499 len 16384 data: 12 11 10 12 03/30 Read block 250 len 12288 04/30 Read block 41 len 16384 05/30 Read block 93 len 4096 06/30 Read block 111 len 20480 07/30 Flush 08/30 Write block 315 len 8192 data: 4 8 09/30 Write block 113 len 24576 data: 10 9 10 11 11 11 10/30 Read block 28 len 16384 11/30 Write block 74 len 24576 data: 6 8 9 7 6 8 12/30 Write block 264 len 4096 data: 5 13/30 Read block 367 len 24576 14/30 Read block 534 len 20480 15/30 Write block 551 len 32768 data: 14 15 17 11 10 8 8 6 16/30 Write block 56 len 24576 data: 9 8 10 9 9 6 17/30 Write block 252 len 28672 data: 16 14 11 15 15 13 9 18/30 Read block 101 len 4096 19/30 Read block 151 len 12288 20/30 Read block 300 len 4096 21/30 Read block 65 len 24576 22/30 Read block 503 len 20480 23/30 Read block 400 len 16384 24/30 Read block 535 len 24576 25/30 Read block 522 len 16384 26/30 Write block 203 len 24576 data: 11 12 12 13 10 9 27/30 Read block 554 len 32768 28/30 Write block 296 len 4096 data: 9 29/30 Read block 369 len 4096 30/30 Read block 251 len 28672 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 2 Done Done Done false 1008 Acked Write 6 Done Done Done false 1009 Acked Read 4 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Write 1 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 8 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 1 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 4 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 8 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 7 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 22 23 23 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:25.027636756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1646} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 114 114 114 204 204 204 T T T 3 060-079 101 101 101 183 183 183 T T T 4 080-099 112 112 112 203 203 203 F F F 5 100-119 107 107 107 197 197 197 T T T 6 120-139 112 112 112 203 203 203 F F F 7 140-159 112 112 112 203 203 203 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 112 112 112 203 203 203 T T T 11 220-239 115 115 115 205 205 205 F F F 12 240-259 115 115 115 205 205 205 T T T 13 260-279 112 112 112 203 203 203 T T T 14 280-299 106 106 106 192 192 192 T T T 15 300-319 102 102 102 185 185 185 T T T 16 320-339 115 115 115 205 205 205 F F F 17 340-359 115 115 115 205 205 205 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 115 115 115 205 205 205 F F F 20 400-419 115 115 115 205 205 205 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 116 116 116 206 206 206 F F F 25 500-519 116 116 116 206 206 206 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 115 115 115 205 205 205 T T T 28 560-579 110 110 110 200 200 200 F F F 29 580-599 115 115 115 205 205 205 F F F Max gen: 116, Max flush: 206 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 112 112 112 203 203 203 F F F 5 100-119 117 117 117 207 207 207 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 112 112 112 203 203 203 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 117 117 117 207 207 207 F F F 11 220-239 115 115 115 205 205 205 F F F 12 240-259 117 117 117 207 207 207 F F F 13 260-279 117 117 117 207 207 207 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 115 115 115 205 205 205 F F F 17 340-359 115 115 115 205 205 205 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 115 115 115 205 205 205 F F F 20 400-419 115 115 115 205 205 205 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 116 116 116 206 206 206 F F F 25 500-519 116 116 116 206 206 206 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 117 117 117 207 207 207 F F F 28 560-579 110 110 110 200 200 200 F F F 29 580-599 115 115 115 205 205 205 F F F Max gen: 117, Max flush: 207 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.392907894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:29.435645081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.436039397Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.467355087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.467888737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.467939645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.467962255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.467986254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} {"msg":"Crucible 4deaf98d-40c8-4729-9283-20330b9d5de3 has session id: f6c282cc-0b20-45c3-8a78-5c09123c3956","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.468452987Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Upstairs opts: Upstairs UUID: 4deaf98d-40c8-4729-9283-20330b9d5de3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.468496695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Crucible stats registered with UUID: 4deaf98d-40c8-4729-9283-20330b9d5de3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.468520795Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.49884182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.498887488Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} {"msg":"4deaf98d-40c8-4729-9283-20330b9d5de3 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.498953276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.498984965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499020023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499049562Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499075512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499094681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.49911262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499139559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499167018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499194247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_connection connected from Ok(127.0.0.1:62873)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499234346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"io task","client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_connection connected from Ok(127.0.0.1:58131)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499345832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"io task","client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_connection connected from Ok(127.0.0.1:59345)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.49940208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"io task","client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499891932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.499921251Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.49993647Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.49995355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.500135993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.500179482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.500623306Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.500666624Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.500700793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501204905Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501246163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501275412Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501297631Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.50132174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.50134597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501369129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501394198Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 207, 207, 203, 207, 203, 203, 204, 203, 207, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501418767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 117, 112, 117, 112, 112, 114, 112, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501444316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501476235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 207, 207, 203, 207, 203, 203, 204, 203, 207, 205]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501504104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 117, 112, 117, 112, 112, 114, 112, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501529453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501557752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 207, 207, 203, 207, 203, 203, 204, 203, 207, 205]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:07:29.501587771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 117, 112, 117, 112, 112, 114, 112, 117, 115]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.50161798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501649029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501677358Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501706077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Next flush: 208","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501733896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"All extents match","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:07:29.501762344Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"":"downstairs","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501790354Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501816463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"0","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501842872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"1","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501860331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"client":"2","":"downstairs","session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501874231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.5018867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"4deaf98d-40c8-4729-9283-20330b9d5de3 is now active with session: f6c282cc-0b20-45c3-8a78-5c09123c3956","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.50189988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501912109Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654,"session_id":"f6c282cc-0b20-45c3-8a78-5c09123c3956"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:29.501924379Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 506 len 28672 data: 10 8 9 8 11 12 10 02/30 Write block 483 len 40960 data: 8 9 9 8 8 8 6 7 8 10 03/30 Read block 539 len 32768 04/30 Write block 544 len 20480 data: 6 7 11 13 16 05/30 Flush 06/30 Write block 84 len 40960 data: 5 7 10 13 14 13 14 10 11 12 07/30 Flush 08/30 Write block 87 len 32768 data: 14 15 14 15 11 12 13 12 09/30 Write block 507 len 36864 data: 9 10 9 12 13 11 8 10 7 10/30 Write block 563 len 12288 data: 5 6 10 11/30 Read block 478 len 4096 12/30 Write block 239 len 4096 data: 8 13/30 Read block 261 len 32768 14/30 Write block 583 len 40960 data: 9 10 11 11 12 13 13 12 8 8 15/30 Write block 574 len 4096 data: 6 16/30 Read block 561 len 16384 17/30 Write block 216 len 12288 data: 12 12 10 18/30 Write block 153 len 20480 data: 5 5 7 8 9 19/30 Flush 20/30 Write block 82 len 16384 data: 7 5 6 8 21/30 Read block 233 len 16384 22/30 Read block 267 len 8192 23/30 Read block 186 len 40960 24/30 Write block 394 len 36864 data: 13 15 15 15 15 14 12 9 7 25/30 Read block 478 len 40960 26/30 Read block 458 len 12288 27/30 Write block 588 len 36864 data: 14 14 13 9 9 9 9 7 7 28/30 Read block 157 len 4096 29/30 Write block 548 len 4096 data: 17 30/30 Write block 109 len 8192 data: 8 9 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 4 Done Done Done false 1020 Acked Read 4 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 9 Done Sent Sent false 1027 Acked Read 1 Done Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 4 4 10 Done 9 7 7 23 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:29.526990404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1654} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 118 118 118 210 210 210 T T T 5 100-119 117 117 117 207 207 207 T T T 6 120-139 112 112 112 203 203 203 F F F 7 140-159 118 118 118 210 210 210 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 118 118 118 210 210 210 F F F 12 240-259 117 117 117 207 207 207 F F F 13 260-279 117 117 117 207 207 207 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 115 115 115 205 205 205 F F F 17 340-359 115 115 115 205 205 205 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 115 115 115 205 205 205 T T T 20 400-419 115 115 115 205 205 205 T T T 21 420-439 115 115 115 205 205 205 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 118 118 118 208 208 208 F F F 25 500-519 118 118 118 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 118 118 118 208 208 208 T T T 28 560-579 118 118 118 210 210 210 F F F 29 580-599 118 118 118 210 210 210 T T T Max gen: 118, Max flush: 210 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 119 119 119 211 211 211 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 118 118 118 210 210 210 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 118 118 118 210 210 210 F F F 12 240-259 117 117 117 207 207 207 F F F 13 260-279 117 117 117 207 207 207 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 115 115 115 205 205 205 F F F 17 340-359 115 115 115 205 205 205 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 119 119 119 211 211 211 F F F 20 400-419 119 119 119 211 211 211 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 111 111 111 202 202 202 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 118 118 118 208 208 208 F F F 25 500-519 118 118 118 210 210 210 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 119 119 119 211 211 211 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 119 119 119 211 211 211 F F F Max gen: 119, Max flush: 211 Begin loop 60 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.880315294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:33.922475962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.922889298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.953722945Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954258555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954300874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954317943Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954335732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} {"msg":"Crucible ab4ba375-1603-4ff5-9e74-fad072e36c29 has session id: 2cae1404-48b3-4290-9905-f5eb10406aa9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954803266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Upstairs opts: Upstairs UUID: ab4ba375-1603-4ff5-9e74-fad072e36c29, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954843854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Crucible stats registered with UUID: ab4ba375-1603-4ff5-9e74-fad072e36c29","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.954860973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.98460094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984631869Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} {"msg":"ab4ba375-1603-4ff5-9e74-fad072e36c29 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984661998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984679097Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984709296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984727315Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984752154Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984783883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984815962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984847641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.98487928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984906629Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61226)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984933258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"io task","client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48438)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.984961427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"io task","client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:48686)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.985013995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"io task","client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.985763428Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.985812196Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.985887173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.985921402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.985956901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.9859887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.986417824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.986456993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.986571559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.986964534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.986997873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987021212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987110999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987131099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987144918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987157087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987170577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 207, 207, 211, 211, 203, 210, 204, 203, 210, 210]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987184746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 117, 119, 119, 112, 118, 114, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987198886Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987217005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 207, 207, 211, 211, 203, 210, 204, 203, 210, 210]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987231445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 117, 119, 119, 112, 118, 114, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987245194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987259104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 207, 207, 211, 211, 203, 210, 204, 203, 210, 210]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987272873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 117, 119, 119, 112, 118, 114, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987286573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987300452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987313632Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987327101ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Next flush: 212","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987348821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.98736192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"":"downstairs","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.98737527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987387399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"0","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987400939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"1","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987416448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"client":"2","":"downstairs","session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987430108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987442607Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"ab4ba375-1603-4ff5-9e74-fad072e36c29 is now active with session: 2cae1404-48b3-4290-9905-f5eb10406aa9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987455517Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987467436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662,"session_id":"2cae1404-48b3-4290-9905-f5eb10406aa9"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:33.987479826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 276 len 4096 data: 11 03/30 Write block 510 len 24576 data: 13 14 12 9 11 8 04/30 Write block 348 len 20480 data: 12 12 7 5 5 05/30 Write block 89 len 24576 data: 15 16 12 13 14 13 06/30 Write block 274 len 4096 data: 11 07/30 Write block 481 len 36864 data: 7 7 9 10 10 9 9 9 7 08/30 Read block 367 len 40960 09/30 Read block 46 len 4096 10/30 Write block 331 len 4096 data: 13 11/30 Read block 351 len 36864 12/30 Read block 355 len 28672 13/30 Write block 440 len 32768 data: 6 6 4 3 4 5 8 10 14/30 Flush 15/30 Write block 325 len 40960 data: 6 6 7 8 8 11 14 14 15 14 16/30 Read block 191 len 12288 17/30 Read block 165 len 28672 18/30 Read block 146 len 28672 19/30 Read block 251 len 12288 20/30 Read block 345 len 24576 21/30 Read block 483 len 24576 22/30 Read block 306 len 28672 23/30 Read block 111 len 8192 24/30 Read block 495 len 36864 25/30 Write block 322 len 40960 data: 10 8 6 7 7 8 9 9 12 15 26/30 Read block 357 len 12288 27/30 Read block 416 len 4096 28/30 Write block 450 len 32768 data: 9 7 8 12 10 10 7 10 29/30 Read block 450 len 12288 30/30 Read block 167 len 4096 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:16 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1014 Acked Write 10 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Read 7 Done Done Done false 1017 Acked Read 7 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Read 6 Done Done Done false 1020 Acked Read 6 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 9 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 16 16 16 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1013 1013 1013 Downstairs last five completed: 1013 1012 1011 1010 1009 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:34.010539703Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1662} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 118 118 118 210 210 210 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 118 118 118 210 210 210 F F F 12 240-259 117 117 117 207 207 207 F F F 13 260-279 120 120 120 213 213 213 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 120 120 120 213 213 213 T T T 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 119 119 119 211 211 211 F F F 20 400-419 119 119 119 211 211 211 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 120 120 120 213 213 213 T T T 23 460-479 115 115 115 205 205 205 F F F 24 480-499 120 120 120 213 213 213 F F F 25 500-519 120 120 120 213 213 213 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 119 119 119 211 211 211 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 119 119 119 211 211 211 F F F Max gen: 120, Max flush: 213 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 117 117 117 207 207 207 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 118 118 118 210 210 210 F F F 8 160-179 114 114 114 204 204 204 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 118 118 118 210 210 210 F F F 12 240-259 117 117 117 207 207 207 F F F 13 260-279 120 120 120 213 213 213 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 114 114 114 204 204 204 F F F 19 380-399 119 119 119 211 211 211 F F F 20 400-419 119 119 119 211 211 211 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 121 121 121 214 214 214 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 120 120 120 213 213 213 F F F 25 500-519 120 120 120 213 213 213 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 119 119 119 211 211 211 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 119 119 119 211 211 211 F F F Max gen: 121, Max flush: 214 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.312906903Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.357117777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.357697826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.357734695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.357749134Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.357763704Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} {"msg":"Crucible 80370000-4cac-4bc2-abb9-91ec7879e9c2 has session id: eafe16f4-cbcc-444f-b7c7-fcb833656373","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.358229577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Upstairs opts: Upstairs UUID: 80370000-4cac-4bc2-abb9-91ec7879e9c2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.358259566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Crucible stats registered with UUID: 80370000-4cac-4bc2-abb9-91ec7879e9c2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.358274615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388424657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388458925Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} {"msg":"80370000-4cac-4bc2-abb9-91ec7879e9c2 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388510374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388530593Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388563972Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.38860376Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388642469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388677878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388709907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388740495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388769654Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388796583Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_connection connected from Ok(127.0.0.1:50171)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388824862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"io task","client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_connection connected from Ok(127.0.0.1:61076)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388907109Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"io task","client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_connection connected from Ok(127.0.0.1:37387)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.388947108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"io task","client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.389427941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.38945736Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.389609844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.389652633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.389667742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.389682101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390066238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390210592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390508351Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390681835Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390707584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390722284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390809121Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.39082744Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.390841059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.39110906Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391127219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 207, 207, 213, 211, 203, 210, 204, 203, 210, 210]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391141549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 117, 120, 119, 112, 118, 114, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391155838Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391170338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 207, 207, 213, 211, 203, 210, 204, 203, 210, 210]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391187827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 117, 120, 119, 112, 118, 114, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391202006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391216316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 207, 207, 213, 211, 203, 210, 204, 203, 210, 210]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391230325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 117, 120, 119, 112, 118, 114, 112, 118, 118]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391244085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391258224Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391271484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391284803Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Next flush: 215","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391297873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391311123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"":"downstairs","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} Wait for a query_work_queue command to finish before sending IO {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391331572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391343952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"0","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391357331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"1","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391370591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"client":"2","":"downstairs","session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.39138389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391398399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"80370000-4cac-4bc2-abb9-91ec7879e9c2 is now active with session: eafe16f4-cbcc-444f-b7c7-fcb833656373","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391411439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391423498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670,"session_id":"eafe16f4-cbcc-444f-b7c7-fcb833656373"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:38.391435888Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 166 len 40960 data: 9 9 8 6 6 6 6 5 5 6 02/30 Read block 173 len 12288 03/30 Read block 529 len 24576 04/30 Write block 509 len 40960 data: 10 14 15 13 10 12 9 8 10 9 05/30 Read block 323 len 36864 06/30 Write block 404 len 40960 data: 5 5 5 6 7 10 11 10 10 10 07/30 Read block 23 len 36864 08/30 Write block 60 len 40960 data: 10 7 10 9 7 7 8 7 6 7 09/30 Read block 597 len 12288 10/30 Read block 455 len 8192 11/30 Read block 529 len 12288 12/30 Read block 232 len 16384 13/30 Write block 597 len 12288 data: 7 7 6 14/30 Read block 399 len 8192 15/30 Write block 229 len 12288 data: 5 6 7 16/30 Read block 428 len 16384 17/30 Read block 502 len 32768 18/30 Write block 489 len 12288 data: 8 8 9 19/30 Write block 148 len 36864 data: 9 7 7 7 8 6 6 8 9 20/30 Write block 453 len 8192 data: 13 11 21/30 Write block 370 len 4096 data: 15 22/30 Write block 237 len 16384 data: 6 6 9 9 23/30 Read block 210 len 40960 24/30 Flush 25/30 Write block 498 len 12288 data: 10 13 12 26/30 Write block 70 len 36864 data: 8 8 6 4 7 9 10 8 7 27/30 Read block 440 len 4096 28/30 Write block 254 len 12288 data: 12 16 16 29/30 Write block 551 len 24576 data: 15 16 18 12 11 9 30/30 Read block 443 len 36864 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Write 6 Done Done Done false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 1 1 Done 6 6 5 17 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:38.413795541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1670} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 122 122 122 215 215 215 T T T 4 080-099 120 120 120 213 213 213 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 122 122 122 215 215 215 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 122 122 122 215 215 215 F F F 12 240-259 122 122 122 215 215 215 T T T 13 260-279 120 120 120 213 213 213 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 122 122 122 215 215 215 F F F 19 380-399 119 119 119 211 211 211 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 122 122 122 215 215 215 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 122 122 122 215 215 215 T T T 25 500-519 122 122 122 215 215 215 T T T 26 520-539 115 115 115 205 205 205 F F F 27 540-559 119 119 119 211 211 211 T T T 28 560-579 118 118 118 210 210 210 F F F 29 580-599 122 122 122 215 215 215 F F F Max gen: 122, Max flush: 215 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 123 123 123 216 216 216 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 119 119 119 211 211 211 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 122 122 122 215 215 215 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 122 122 122 215 215 215 F F F 12 240-259 123 123 123 216 216 216 F F F 13 260-279 120 120 120 213 213 213 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 117 117 117 207 207 207 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 120 120 120 213 213 213 F F F 18 360-379 122 122 122 215 215 215 F F F 19 380-399 119 119 119 211 211 211 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 115 115 115 205 205 205 F F F 22 440-459 122 122 122 215 215 215 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 123 123 123 216 216 216 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 123 123 123 216 216 216 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 122 122 122 215 215 215 F F F Max gen: 123, Max flush: 216 Begin loop 62 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.725827022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.76909124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.769712418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.769756546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.769779285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.769797985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} {"msg":"Crucible 28ee766c-6fa6-4811-890b-2d87a5087f83 has session id: 43f76b7d-282e-49dc-ac1c-2373479ad837","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.770228509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Upstairs opts: Upstairs UUID: 28ee766c-6fa6-4811-890b-2d87a5087f83, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.770274208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Crucible stats registered with UUID: 28ee766c-6fa6-4811-890b-2d87a5087f83","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.770295447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800010293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800045362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} {"msg":"28ee766c-6fa6-4811-890b-2d87a5087f83 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800081001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80009821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800125409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800143019Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800160178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800186607Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800228586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800262115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800296413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800325192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_connection connected from Ok(127.0.0.1:55081)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800345181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"io task","client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_connection connected from Ok(127.0.0.1:41385)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80037797Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"io task","client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_connection connected from Ok(127.0.0.1:62483)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.800411209Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"io task","client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801127763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801168252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801198381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80122811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801259328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801277558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801836168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801870497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.801899595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802410767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802440686Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802456445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802510053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802551382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802570031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802583151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80259672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 207, 216, 213, 211, 203, 215, 215, 203, 210, 215]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80261085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 123, 120, 119, 112, 122, 122, 112, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802625099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802639099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 207, 216, 213, 211, 203, 215, 215, 203, 210, 215]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802653848Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 123, 120, 119, 112, 122, 122, 112, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802667548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802681497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 207, 216, 213, 211, 203, 215, 215, 203, 210, 215]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802699356Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 123, 120, 119, 112, 122, 122, 112, 118, 122]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802713636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802727546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802740455Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802753434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Next flush: 217","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802766404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"All extents match","v":0,"name"Wait for a query_work_queue command to finish before sending IO :"crucible","level":30,"time":"2024-11-07T14:07:42.802791393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"":"downstairs","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802804893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802816932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"0","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802830022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"1","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802843051Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"client":"2","":"downstairs","session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802856241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80286846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"28ee766c-6fa6-4811-890b-2d87a5087f83 is now active with session: 43f76b7d-282e-49dc-ac1c-2373479ad837","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80288122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.80289336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678,"session_id":"43f76b7d-282e-49dc-ac1c-2373479ad837"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:42.802907439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 377 len 28672 data: 12 12 13 13 15 14 14 02/30 Read block 413 len 28672 03/30 Flush 04/30 Write block 593 len 28672 data: 10 10 8 8 8 8 7 05/30 Read block 304 len 28672 06/30 Write block 374 len 12288 data: 13 13 12 07/30 Read block 263 len 8192 08/30 Read block 31 len 28672 09/30 Read block 551 len 20480 10/30 Read block 22 len 36864 11/30 Read block 189 len 8192 12/30 Read block 566 len 20480 13/30 Write block 223 len 28672 data: 7 6 5 5 6 5 6 14/30 Write block 312 len 24576 data: 6 6 6 5 9 11 15/30 Write block 244 len 16384 data: 12 12 11 9 16/30 Write block 144 len 32768 data: 7 11 12 13 10 8 8 8 17/30 Flush 18/30 Write block 260 len 40960 data: 7 6 6 5 6 8 9 10 7 7 19/30 Read block 147 len 28672 20/30 Flush 21/30 Write block 246 len 20480 data: 12 10 11 12 12 22/30 Read block 400 len 36864 23/30 Read block 195 len 40960 24/30 Write block 380 len 24576 data: 14 16 15 15 14 15 25/30 Write block 106 len 40960 data: 12 9 10 9 10 10 11 11 10 11 26/30 Read block 267 len 40960 27/30 Read block 185 len 24576 28/30 Write block 426 len 16384 data: 9 9 7 8 29/30 Write block 340 len 20480 data: 9 12 14 14 13 30/30 Read block 184 len 28672 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 10 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 5 Sent Done Done false 1029 Acked Read 7 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 0 3 Done 8 9 10 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:42.828576502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1678} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 123 123 123 216 216 216 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 119 119 119 211 211 211 T T T 6 120-139 112 112 112 203 203 203 F F F 7 140-159 124 124 124 218 218 218 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 124 124 124 218 218 218 F F F 12 240-259 124 124 124 218 218 218 T T T 13 260-279 124 124 124 219 219 219 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 124 124 124 218 218 218 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 120 120 120 213 213 213 T T T 18 360-379 124 124 124 218 218 218 F F F 19 380-399 124 124 124 217 217 217 T T T 20 400-419 122 122 122 215 215 215 F F F 21 420-439 115 115 115 205 205 205 T T T 22 440-459 122 122 122 215 215 215 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 123 123 123 216 216 216 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 123 123 123 216 216 216 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 124 124 124 218 218 218 F F F Max gen: 124, Max flush: 219 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 F F F 1 020-039 115 115 115 205 205 205 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 123 123 123 216 216 216 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 125 125 125 220 220 220 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 124 124 124 218 218 218 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 F F F 11 220-239 124 124 124 218 218 218 F F F 12 240-259 125 125 125 220 220 220 F F F 13 260-279 124 124 124 219 219 219 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 124 124 124 218 218 218 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 125 125 125 220 220 220 F F F 18 360-379 124 124 124 218 218 218 F F F 19 380-399 125 125 125 220 220 220 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 125 125 125 220 220 220 F F F 22 440-459 122 122 122 215 215 215 F F F 23 460-479 115 115 115 205 205 205 F F F 24 480-499 123 123 123 216 216 216 F F F 25 500-519 123 123 123 216 216 216 F F F 26 520-539 115 115 115 205 205 205 F F F 27 540-559 123 123 123 216 216 216 F F F 28 560-579 118 118 118 210 210 210 F F F 29 580-599 124 124 124 218 218 218 F F F Max gen: 125, Max flush: 220 Begin loop 63 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.138600295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.181994459Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.182565778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.182604446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.182619276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.182638405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} {"msg":"Crucible 15ea2077-73ce-4c1b-9846-512dbae40176 has session id: 7613f697-2e6a-416c-8b3b-c7790e926766","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.183108588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Upstairs opts: Upstairs UUID: 15ea2077-73ce-4c1b-9846-512dbae40176, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.183152787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Crucible stats registered with UUID: 15ea2077-73ce-4c1b-9846-512dbae40176","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.183176406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213092836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213126015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} {"msg":"15ea2077-73ce-4c1b-9846-512dbae40176 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213197042Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213219311Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.21324776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.2132658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213283139Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213299909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213316698Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213333327Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213364916Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213403585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_connection connected from Ok(127.0.0.1:43130)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213708774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"io task","client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_connection connected from Ok(127.0.0.1:51950)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213794071Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"io task","client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_connection connected from Ok(127.0.0.1:61802)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.213828679Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"io task","client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214155338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214194226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214226455Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214260424Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214334471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.21435443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214871122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.214977058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215040896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215427182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215453941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.21547179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215544217Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215578386Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215594706Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215607585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215621685Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[0]R flush_numbers[0..12]: [205, 205, 207, 216, 213, 220, 203, 218, 215, 203, 210, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215636294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[0]R generation[0..12]: [115, 115, 117, 123, 120, 125, 112, 124, 122, 112, 118, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215650284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215664463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[1]R flush_numbers[0..12]: [205, 205, 207, 216, 213, 220, 203, 218, 215, 203, 210, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215678563Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[1]R generation[0..12]: [115, 115, 117, 123, 120, 125, 112, 124, 122, 112, 118, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215692462Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215706711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[2]R flush_numbers[0..12]: [205, 205, 207, 216, 213, 220, 203, 218, 215, 203, 210, 218]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215720701Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[2]R generation[0..12]: [115, 115, 117, 123, 120, 125, 112, 124, 122, 112, 118, 124]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215734501Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.21575665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Max found gen is 126","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:07:47.215781989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Generation requested: 126 >= found:126","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215808008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Next flush: 221","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215829367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215843027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"":"downstairs","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215856336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215868496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"0","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215881815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"1","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215895005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"client":"2","":"downstairs","session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215908494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215920924Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"15ea2077-73ce-4c1b-9846-512dbae40176 is now active with session: 7613f697-2e6a-416c-8b3b-c7790e926766","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215933893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215945953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686,"session_id":"7613f697-2e6a-416c-8b3b-c7790e926766"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:47.215958302Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 462 len 28672 data: 14 13 10 11 10 10 8 02/30 Read block 23 len 12288 03/30 Read block 98 len 32768 04/30 Write block 571 len 12288 data: 12 9 8 05/30 Read block 309 len 40960 06/30 Write block 142 len 36864 data: 4 6 8 12 13 14 11 9 9 07/30 Read block 478 len 32768 08/30 Write block 568 len 16384 data: 11 12 12 13 09/30 Write block 0 len 32768 data: 2 3 3 4 5 5 6 5 10/30 Write block 570 len 4096 data: 13 11/30 Write block 437 len 8192 data: 7 7 12/30 Write block 448 len 28672 data: 8 9 10 8 9 14 12 13/30 Read block 193 len 36864 14/30 Read block 541 len 24576 15/30 Read block 248 len 20480 16/30 Write block 494 len 32768 data: 13 11 13 13 11 14 13 11 17/30 Write block 31 len 36864 data: 15 11 10 13 10 10 12 14 12 18/30 Read block 229 len 12288 19/30 Read block 467 len 12288 20/30 Write block 113 len 20480 data: 12 11 12 12 12 21/30 Write block 235 len 40960 data: 4 5 7 7 10 10 6 8 9 13 22/30 Write block 225 len 32768 data: 6 6 7 6 7 7 8 6 23/30 Write block 203 len 12288 data: 12 13 13 24/30 Read block 563 len 12288 25/30 Read block 190 len 28672 26/30 Write block 69 len 24576 data: 8 9 9 7 5 8 27/30 Read block 510 len 12288 28/30 Read block 173 len 36864 29/30 Read block 309 len 20480 30/30 Write block 526 len 28672 data: 9 7 6 7 8 11 10 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Read 10 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 4 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 2 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 6 Done Done Done false 1014 Acked Read 5 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 3 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 10 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 29 29 29 87 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:47.23569947Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1686} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 115 115 115 205 205 205 T T T 1 020-039 115 115 115 205 205 205 T T T 2 040-059 117 117 117 207 207 207 F F F 3 060-079 123 123 123 216 216 216 T T T 4 080-099 120 120 120 213 213 213 F F F 5 100-119 125 125 125 220 220 220 T T T 6 120-139 112 112 112 203 203 203 F F F 7 140-159 124 124 124 218 218 218 T T T 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 118 118 118 210 210 210 T T T 11 220-239 124 124 124 218 218 218 T T T 12 240-259 125 125 125 220 220 220 T T T 13 260-279 124 124 124 219 219 219 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 124 124 124 218 218 218 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 125 125 125 220 220 220 F F F 18 360-379 124 124 124 218 218 218 F F F 19 380-399 125 125 125 220 220 220 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 125 125 125 220 220 220 T T T 22 440-459 122 122 122 215 215 215 T T T 23 460-479 115 115 115 205 205 205 T T T 24 480-499 123 123 123 216 216 216 T T T 25 500-519 123 123 123 216 216 216 T T T 26 520-539 115 115 115 205 205 205 T T T 27 540-559 123 123 123 216 216 216 F F F 28 560-579 118 118 118 210 210 210 T T T 29 580-599 124 124 124 218 218 218 F F F Max gen: 125, Max flush: 220 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 221 221 221 F F F 1 020-039 126 126 126 221 221 221 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 126 126 126 221 221 221 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 126 126 126 221 221 221 F F F 6 120-139 112 112 112 203 203 203 F F F 7 140-159 126 126 126 221 221 221 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 126 126 126 221 221 221 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 124 124 124 219 219 219 F F F 14 280-299 117 117 117 207 207 207 F F F 15 300-319 124 124 124 218 218 218 F F F 16 320-339 121 121 121 214 214 214 F F F 17 340-359 125 125 125 220 220 220 F F F 18 360-379 124 124 124 218 218 218 F F F 19 380-399 125 125 125 220 220 220 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 126 126 126 221 221 221 F F F 22 440-459 126 126 126 221 221 221 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 126 126 126 221 221 221 F F F 26 520-539 126 126 126 221 221 221 F F F 27 540-559 123 123 123 216 216 216 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 124 124 124 218 218 218 F F F Max gen: 126, Max flush: 221 Begin loop 64 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.684259702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.727775661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.732189242Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.73222629Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.73224454Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.732259479Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} {"msg":"Crucible 15eb2add-eda2-43fa-989f-4f3cdf062d79 has session id: 6aef18bf-eb8c-4942-bc7c-f51401fb7887","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.732681914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Upstairs opts: Upstairs UUID: 15eb2add-eda2-43fa-989f-4f3cdf062d79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.732717433Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Crucible stats registered with UUID: 15eb2add-eda2-43fa-989f-4f3cdf062d79","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.732733222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762661222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76270805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} {"msg":"15eb2add-eda2-43fa-989f-4f3cdf062d79 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762746329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762777687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762825496Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762862735Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762895423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762927732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.762953581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76298199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763015559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763050138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_connection connected from Ok(127.0.0.1:54879)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763088046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"io task","client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_connection connected from Ok(127.0.0.1:46438)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763125045Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"io task","client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_connection connected from Ok(127.0.0.1:40879)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763161844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"io task","client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763848619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763879168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763895077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.763909217Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76409682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.764145388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.764590862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.764623221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76465239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765147462Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765175981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76519113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765255048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765274617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765298676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765311536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765325075Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[0]R flush_numbers[0..12]: [221, 221, 207, 221, 213, 221, 203, 221, 215, 203, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765339355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[0]R generation[0..12]: [126, 126, 117, 126, 120, 126, 112, 126, 122, 112, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765353534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765367954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[1]R flush_numbers[0..12]: [221, 221, 207, 221, 213, 221, 203, 221, 215, 203, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765382064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[1]R generation[0..12]: [126, 126, 117, 126, 120, 126, 112, 126, 122, 112, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765396013Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765410952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[2]R flush_numbers[0..12]: [221, 221, 207, 221, 213, 221, 203, 221, 215, 203, 221, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765424922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[2]R generation[0..12]: [126, 126, 117, 126, 120, 126, 112, 126, 122, 112, 126, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765441841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765456221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Max found gen is 127","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76546927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Generation requested: 128 >= found:127","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.76548279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765495929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765508869Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"":"downstairs","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765522228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765534238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"0","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765558527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"1","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765572127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"client":"2","":"downstairs","session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765585446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765597856Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"15eb2add-eda2-43fa-989f-4f3cdf062d79 is now active with session: 6aef18bf-eb8c-4942-bc7c-f51401fb7887","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765610625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765622775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694,"session_id":"6aef18bf-eb8c-4942-bc7c-f51401fb7887"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:51.765635184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 112 len 32768 data: 12 13 12 13 13 13 12 13 02/30 Read block 581 len 36864 03/30 Write block 513 len 24576 data: 11 13 10 9 11 10 04/30 Read block 347 len 4096 05/30 Write block 279 len 8192 data: 11 12 06/30 Read block 331 len 24576 07/30 Read block 176 len 16384 08/30 Write block 598 len 4096 data: 9 09/30 Write block 428 len 40960 data: 8 9 7 7 9 9 9 8 8 8 10/30 Read block 56 len 8192 11/30 Write block 334 len 8192 data: 15 15 12/30 Read block 215 len 20480 13/30 Read block 585 len 28672 14/30 Write block 267 len 32768 data: 11 8 8 9 11 12 10 12 15/30 Write block 76 len 8192 data: 11 9 16/30 Read block 487 len 16384 17/30 Write block 449 len 28672 data: 10 11 9 10 15 13 11 18/30 Write block 441 len 4096 data: 7 19/30 Write block 503 len 4096 data: 12 20/30 Write block 425 len 4096 data: 11 21/30 Flush 22/30 Write block 125 len 32768 data: 13 16 17 18 14 14 14 15 23/30 Read block 541 len 4096 24/30 Read block 142 len 8192 25/30 Write block 546 len 32768 data: 12 14 18 16 15 16 17 19 26/30 Flush 27/30 Write block 273 len 4096 data: 11 28/30 Read block 311 len 40960 29/30 Write block 213 len 28672 data: 8 6 8 13 13 11 10 30/30 Read block 413 len 32768 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 7 Done Done Done false 1029 Acked Read 8 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 0 2 Done 3 3 4 10 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:51.788399102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1694} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 221 221 221 F F F 1 020-039 126 126 126 221 221 221 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 128 128 128 222 222 222 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 128 128 128 222 222 222 F F F 6 120-139 128 128 128 223 223 223 F F F 7 140-159 126 126 126 221 221 221 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 126 126 126 221 221 221 T T T 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 128 128 128 222 222 222 T T T 14 280-299 128 128 128 222 222 222 F F F 15 300-319 124 124 124 218 218 218 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 125 125 125 220 220 220 F F F 18 360-379 124 124 124 218 218 218 F F F 19 380-399 125 125 125 220 220 220 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 128 128 128 222 222 222 F F F 22 440-459 128 128 128 222 222 222 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 222 222 222 F F F 26 520-539 126 126 126 221 221 221 F F F 27 540-559 128 128 128 223 223 223 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 128 128 128 222 222 222 F F F Max gen: 128, Max flush: 223 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 221 221 221 F F F 1 020-039 126 126 126 221 221 221 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 128 128 128 222 222 222 F F F 4 080-099 120 120 120 213 213 213 F F F 5 100-119 128 128 128 222 222 222 F F F 6 120-139 128 128 128 223 223 223 F F F 7 140-159 126 126 126 221 221 221 F F F 8 160-179 122 122 122 215 215 215 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 129 129 129 224 224 224 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 129 129 129 224 224 224 F F F 14 280-299 128 128 128 222 222 222 F F F 15 300-319 124 124 124 218 218 218 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 125 125 125 220 220 220 F F F 18 360-379 124 124 124 218 218 218 F F F 19 380-399 125 125 125 220 220 220 F F F 20 400-419 122 122 122 215 215 215 F F F 21 420-439 128 128 128 222 222 222 F F F 22 440-459 128 128 128 222 222 222 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 222 222 222 F F F 26 520-539 126 126 126 221 221 221 F F F 27 540-559 128 128 128 223 223 223 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 128 128 128 222 222 222 F F F Max gen: 129, Max flush: 224 Begin loop 65 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.064042007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.107331265Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.10802079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.108066968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.108087137Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.108105757Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} {"msg":"Crucible a1938f8f-6e06-46c9-ab2c-19dcc14d1adf has session id: 65fefb69-627a-44e8-a287-8234b3286c7e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.108513322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Upstairs opts: Upstairs UUID: a1938f8f-6e06-46c9-ab2c-19dcc14d1adf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.10855709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Crucible stats registered with UUID: a1938f8f-6e06-46c9-ab2c-19dcc14d1adf","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.1085812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138716482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.1387507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} {"msg":"a1938f8f-6e06-46c9-ab2c-19dcc14d1adf active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138821308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138844807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138872266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138890235Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138907545Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138924114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138941033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138959743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.138991822Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.13902784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33922)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139066629Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"io task","client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:33990)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139279791Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"io task","client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_connection connected from Ok(127.0.0.1:39333)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.13932929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"io task","client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139794483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139837361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.13986952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139896679Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139914358Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.139928918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.140498527Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.140527966Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.140557845Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141130985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141159153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141174113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141187112Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141203742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141217541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141230341Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.14124389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[0]R flush_numbers[0..12]: [221, 221, 207, 222, 213, 222, 223, 221, 215, 203, 224, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.14125787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[0]R generation[0..12]: [126, 126, 117, 128, 120, 128, 128, 126, 122, 112, 129, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141272029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141286999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[1]R flush_numbers[0..12]: [221, 221, 207, 222, 213, 222, 223, 221, 215, 203, 224, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141301008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[1]R generation[0..12]: [126, 126, 117, 128, 120, 128, 128, 126, 122, 112, 129, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141314858Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141329047Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[2]R flush_numbers[0..12]: [221, 221, 207, 222, 213, 222, 223, 221, 215, 203, 224, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141343017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[2]R generation[0..12]: [126, 126, 117, 128, 120, 128, 128, 126, 122, 112, 129, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141356976Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141371196Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141384285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141409754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Next flush: 225","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141423214Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141436634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"":"downstairs","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141449753Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141461942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"0","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141475232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"1","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141488772Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"client":"2","":"downstairs","session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141502431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141514971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"a1938f8f-6e06-46c9-ab2c-19dcc14d1adf is now active with session: 65fefb69-627a-44e8-a287-8234b3286c7e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.1415279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.14154Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702,"session_id":"65fefb69-627a-44e8-a287-8234b3286c7e"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:07:56.141552559Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 367 len 4096 data: 9 03/30 Write block 287 len 20480 data: 16 15 12 9 11 04/30 Write block 124 len 16384 data: 12 14 17 18 05/30 Write block 96 len 8192 data: 9 7 06/30 Read block 175 len 16384 07/30 Write block 414 len 28672 data: 9 8 8 7 8 8 11 08/30 Write block 89 len 40960 data: 16 17 13 14 15 14 11 10 8 4 09/30 Read block 363 len 32768 10/30 Read block 126 len 28672 11/30 Write block 121 len 28672 data: 13 12 12 13 15 18 19 12/30 Write block 85 len 28672 data: 9 11 15 16 17 18 14 13/30 Read block 5 len 12288 14/30 Write block 296 len 40960 data: 10 10 12 10 6 7 7 9 7 9 15/30 Read block 47 len 4096 16/30 Read block 449 len 16384 17/30 Read block 455 len 4096 18/30 Write block 106 len 24576 data: 13 10 11 10 11 11 19/30 Write block 354 len 16384 data: 7 9 9 9 20/30 Read block 573 len 40960 21/30 Write block 364 len 24576 data: 12 9 9 10 9 13 22/30 Write block 266 len 32768 data: 10 12 9 9 10 12 13 12 23/30 Write block 22 len 12288 data: 14 10 11 24/30 Read block 416 len 16384 25/30 Write block 165 len 8192 data: 10 10 26/30 Read block 581 len 40960 27/30 Read block 435 len 8192 28/30 Read block 137 len 40960 29/30 Read block 102 len 12288 30/30 Write block 0 len 20480 data: 3 4 4 5 6 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 1 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 4 Done Done Done false 1004 Acked Write 2 Done Done Done false 1005 Acked Read 4 Done Done Done false 1006 Acked Write 7 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 8 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Write 7 Done Done Done false 1011 Acked Write 7 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Write 10 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 4 Done Done Done false 1016 Acked Read 1 Done Done Done false 1017 Acked Write 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 8 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:07:56.160513935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1702} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 221 221 221 T T T 1 020-039 126 126 126 221 221 221 T T T 2 040-059 117 117 117 207 207 207 F F F 3 060-079 128 128 128 222 222 222 F F F 4 080-099 120 120 120 213 213 213 T T T 5 100-119 128 128 128 222 222 222 T T T 6 120-139 128 128 128 223 223 223 T T T 7 140-159 126 126 126 221 221 221 F F F 8 160-179 122 122 122 215 215 215 T T T 9 180-199 112 112 112 203 203 203 F F F 10 200-219 129 129 129 224 224 224 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 129 129 129 224 224 224 T T T 14 280-299 128 128 128 222 222 222 T T T 15 300-319 124 124 124 218 218 218 T T T 16 320-339 128 128 128 222 222 222 F F F 17 340-359 125 125 125 220 220 220 T T T 18 360-379 124 124 124 218 218 218 T T T 19 380-399 125 125 125 220 220 220 F F F 20 400-419 122 122 122 215 215 215 T T T 21 420-439 128 128 128 222 222 222 T T T 22 440-459 128 128 128 222 222 222 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 222 222 222 F F F 26 520-539 126 126 126 221 221 221 F F F 27 540-559 128 128 128 223 223 223 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 128 128 128 222 222 222 F F F Max gen: 129, Max flush: 224 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 225 225 225 F F F 1 020-039 130 130 130 225 225 225 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 128 128 128 222 222 222 F F F 4 080-099 130 130 130 225 225 225 F F F 5 100-119 130 130 130 225 225 225 F F F 6 120-139 130 130 130 225 225 225 F F F 7 140-159 126 126 126 221 221 221 F F F 8 160-179 130 130 130 225 225 225 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 129 129 129 224 224 224 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 130 130 130 225 225 225 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 130 130 130 225 225 225 F F F 18 360-379 130 130 130 225 225 225 F F F 19 380-399 125 125 125 220 220 220 F F F 20 400-419 130 130 130 225 225 225 F F F 21 420-439 130 130 130 225 225 225 F F F 22 440-459 128 128 128 222 222 222 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 222 222 222 F F F 26 520-539 126 126 126 221 221 221 F F F 27 540-559 128 128 128 223 223 223 F F F 28 560-579 126 126 126 221 221 221 F F F 29 580-599 128 128 128 222 222 222 F F F Max gen: 130, Max flush: 225 Begin loop 66 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.581248011Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.625590341Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626294935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626333454Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626348673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626367153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} {"msg":"Crucible 127de56e-a4bc-440c-a50e-f1ab33b00870 has session id: ec1573b0-6715-4f72-841d-53448ad5cfde","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626735809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Upstairs opts: Upstairs UUID: 127de56e-a4bc-440c-a50e-f1ab33b00870, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626776398Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Crucible stats registered with UUID: 127de56e-a4bc-440c-a50e-f1ab33b00870","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.626802507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657293136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657326985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} {"msg":"127de56e-a4bc-440c-a50e-f1ab33b00870 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657404812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657428081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.6574547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.65747257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657489759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657506598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657523718Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657553027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657585806Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657623224Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_connection connected from Ok(127.0.0.1:38568)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657658083Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"io task","client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_connection connected from Ok(127.0.0.1:42598)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657692122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"io task","client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_connection connected from Ok(127.0.0.1:47266)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.657728701Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"io task","client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.658395726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.658427555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.658441935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.658456434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.658652587Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.658691026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659182138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659213017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659231416Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659719589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659750078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659777067Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659809615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659838034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659867493Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659893423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659916922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[0]R flush_numbers[0..12]: [225, 225, 207, 222, 225, 225, 225, 221, 225, 203, 224, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659942011Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[0]R generation[0..12]: [130, 130, 117, 128, 130, 130, 130, 126, 130, 112, 129, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.65996771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.659994589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[1]R flush_numbers[0..12]: [225, 225, 207, 222, 225, 225, 225, 221, 225, 203, 224, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660019488Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[1]R generation[0..12]: [130, 130, 117, 128, 130, 130, 130, 126, 130, 112, 129, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660043707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660070226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[2]R flush_numbers[0..12]: [225, 225, 207, 222, 225, 225, 225, 221, 225, 203, 224, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660098955Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[2]R generation[0..12]: [130, 130, 117, 128, 130, 130, 130, 126, 130, 112, 129, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660125274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660152053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Max found gen is 131","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2024-11-07T14:08:00.660192252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Generation requested: 132 >= found:131","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660224891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Next flush: 226","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660254839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660284178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"":"downstairs","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660313397Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660339416Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"0","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660367305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"1","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660398954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"client":"2","":"downstairs","session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660424643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660438343Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"127de56e-a4bc-440c-a50e-f1ab33b00870 is now active with session: ec1573b0-6715-4f72-841d-53448ad5cfde","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660463602Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660476791Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710,"session_id":"ec1573b0-6715-4f72-841d-53448ad5cfde"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:00.660489661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 77 len 8192 data: 10 8 02/30 Write block 353 len 24576 data: 5 8 10 10 10 10 03/30 Read block 260 len 4096 04/30 Read block 204 len 32768 05/30 Read block 140 len 20480 06/30 Write block 37 len 4096 data: 13 07/30 Read block 232 len 16384 08/30 Write block 8 len 36864 data: 4 4 4 4 4 8 9 8 10 09/30 Write block 140 len 40960 data: 7 4 5 7 9 13 14 15 12 10 10/30 Read block 285 len 28672 11/30 Read block 448 len 24576 12/30 Read block 189 len 20480 13/30 Read block 75 len 8192 14/30 Write block 530 len 24576 data: 9 12 11 9 10 9 15/30 Read block 536 len 32768 16/30 Write block 86 len 32768 data: 12 16 17 18 19 15 15 16 17/30 Write block 144 len 20480 data: 10 14 15 16 13 18/30 Read block 234 len 16384 19/30 Read block 566 len 8192 20/30 Read block 54 len 40960 21/30 Read block 95 len 32768 22/30 Write block 398 len 20480 data: 16 15 13 10 8 23/30 Read block 223 len 28672 24/30 Write block 568 len 8192 data: 12 13 25/30 Write block 201 len 32768 data: 6 9 13 14 14 14 11 10 26/30 Read block 261 len 20480 27/30 Write block 142 len 24576 data: 6 8 11 15 16 17 28/30 Write block 443 len 32768 data: 4 5 6 9 11 9 11 12 29/30 Write block 152 len 20480 data: 9 7 7 9 10 30/30 Write block 143 len 40960 data: 9 12 16 17 18 14 11 10 9 10 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 9 Done Done Done false 1008 Acked Write 10 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 6 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Write 6 Done Done Done false 1014 Acked Read 8 Done Done Done false 1015 Acked Write 8 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 10 Done Done Done false 1020 Acked Read 8 Done Done Done false 1021 Acked Write 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Write 2 Done Done Done false 1024 Acked Write 8 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Write 6 Sent Sent Sent false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 4 4 4 12 Done 26 26 26 78 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:00.679368149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1710} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 130 130 130 225 225 225 T T T 1 020-039 130 130 130 225 225 225 T T T 2 040-059 117 117 117 207 207 207 F F F 3 060-079 128 128 128 222 222 222 T T T 4 080-099 130 130 130 225 225 225 T T T 5 100-119 130 130 130 225 225 225 F F F 6 120-139 130 130 130 225 225 225 F F F 7 140-159 126 126 126 221 221 221 T T T 8 160-179 130 130 130 225 225 225 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 129 129 129 224 224 224 T T T 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 130 130 130 225 225 225 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 130 130 130 225 225 225 T T T 18 360-379 130 130 130 225 225 225 F F F 19 380-399 125 125 125 220 220 220 T T T 20 400-419 130 130 130 225 225 225 T T T 21 420-439 130 130 130 225 225 225 F F F 22 440-459 128 128 128 222 222 222 T T T 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 222 222 222 F F F 26 520-539 126 126 126 221 221 221 T T T 27 540-559 128 128 128 223 223 223 F F F 28 560-579 126 126 126 221 221 221 T T T 29 580-599 128 128 128 222 222 222 F F F Max gen: 130, Max flush: 225 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 226 226 226 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 117 117 117 207 207 207 F F F 3 060-079 131 131 131 226 226 226 F F F 4 080-099 131 131 131 226 226 226 F F F 5 100-119 130 130 130 225 225 225 F F F 6 120-139 130 130 130 225 225 225 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 130 130 130 225 225 225 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 F F F 13 260-279 130 130 130 225 225 225 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 131 131 131 226 226 226 F F F 18 360-379 130 130 130 225 225 225 F F F 19 380-399 131 131 131 226 226 226 F F F 20 400-419 131 131 131 226 226 226 F F F 21 420-439 130 130 130 225 225 225 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 126 126 126 221 221 221 F F F 24 480-499 126 126 126 221 221 221 F F F 25 500-519 128 128 128 222 222 222 F F F 26 520-539 131 131 131 226 226 226 F F F 27 540-559 128 128 128 223 223 223 F F F 28 560-579 131 131 131 226 226 226 F F F 29 580-599 128 128 128 222 222 222 F F F Max gen: 131, Max flush: 226 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.090118896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:05.132159539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.132575284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.163671861Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.164204932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.16424335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.16426023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.164277889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} {"msg":"Crucible 0b2228e6-30f3-4159-a99e-eeb39d06de56 has session id: 2009852c-f3e6-4c9c-915c-f7ce23961b9e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.164760742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Upstairs opts: Upstairs UUID: 0b2228e6-30f3-4159-a99e-eeb39d06de56, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.16480978Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Crucible stats registered with UUID: 0b2228e6-30f3-4159-a99e-eeb39d06de56","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.164835279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.194978211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195035489Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} {"msg":"0b2228e6-30f3-4159-a99e-eeb39d06de56 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195056718Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195073478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195100697Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195132826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195167854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195203293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195243842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.1952827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195317429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195353458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36319)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195490602Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"io task","client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64250)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.195537641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"io task","client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49118)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.19557747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"io task","client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.19611442Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196157109Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196297113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196326152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196342242Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196363781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196743617Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.196787136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197048276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197320276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197349275Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197364325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197382744Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197396754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197410263Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197546098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197566018Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[0]R flush_numbers[0..12]: [226, 226, 207, 226, 226, 225, 225, 226, 225, 203, 226, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197580247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[0]R generation[0..12]: [131, 131, 117, 131, 131, 130, 130, 131, 130, 112, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197594437Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197608626Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[1]R flush_numbers[0..12]: [226, 226, 207, 226, 226, 225, 225, 226, 225, 203, 226, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197622686Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[1]R generation[0..12]: [131, 131, 117, 131, 131, 130, 130, 131, 130, 112, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197636525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197653835Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[2]R flush_numbers[0..12]: [226, 226, 207, 226, 226, 225, 225, 226, 225, 203, 226, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197667914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[2]R generation[0..12]: [131, 131, 117, 131, 131, 130, 130, 131, 130, 112, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197681603Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197695623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197708802Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Generation requested: 134 >= found:132","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197722082Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197734952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197747971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"":"downstairs","":"downstairsWait for a query_work_queue command to finish before sending IO ","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.19776982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.19778216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"0","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197795429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"1","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197808609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"client":"2","":"downstairs","session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197821909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197834238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"0b2228e6-30f3-4159-a99e-eeb39d06de56 is now active with session: 2009852c-f3e6-4c9c-915c-f7ce23961b9e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197847048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197861507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718,"session_id":"2009852c-f3e6-4c9c-915c-f7ce23961b9e"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:05.197877347Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 430 len 4096 data: 8 02/30 Write block 554 len 4096 data: 13 03/30 Read block 411 len 28672 04/30 Read block 101 len 20480 05/30 Write block 248 len 28672 data: 12 13 13 16 17 15 13 06/30 Write block 360 len 16384 data: 13 12 12 13 07/30 Write block 105 len 36864 data: 13 14 11 12 11 12 12 13 14 08/30 Read block 17 len 4096 09/30 Read block 522 len 40960 10/30 Read block 96 len 4096 11/30 Read block 351 len 20480 12/30 Read block 579 len 24576 13/30 Write block 350 len 32768 data: 8 6 6 6 9 11 11 11 14/30 Read block 301 len 36864 15/30 Read block 228 len 36864 16/30 Read block 281 len 4096 17/30 Write block 1 len 36864 data: 5 5 6 7 6 7 6 5 5 18/30 Read block 392 len 4096 19/30 Read block 585 len 32768 20/30 Write block 481 len 28672 data: 8 8 10 11 11 10 10 21/30 Write block 481 len 16384 data: 9 9 11 12 22/30 Write block 516 len 4096 data: 10 23/30 Read block 545 len 32768 24/30 Write block 463 len 32768 data: 14 11 12 11 11 9 9 9 25/30 Write block 405 len 40960 data: 6 6 7 8 11 12 11 11 11 10 26/30 Write block 589 len 24576 data: 15 14 10 10 11 11 27/30 Write block 57 len 20480 data: 9 11 10 11 8 28/30 Read block 111 len 16384 29/30 Read block 445 len 4096 30/30 Read block 305 len 16384 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 1 Done Done Done false 1002 Acked Read 7 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 7 Done Done Done false 1005 Acked Write 4 Done Done Done false 1006 Acked Write 9 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 1 Done Done Done false 1010 Acked Read 5 Done Done Done false 1011 Acked Read 6 Done Done Done false 1012 Acked Write 8 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 9 Done Done Done false 1015 Acked Read 1 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 1 Done Done Done false 1018 Acked Read 8 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 4 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Read 8 Done Done Done false 1023 Acked Write 8 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Read 4 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 30 30 30 90 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 0 0 0 Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:05.216949348Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1718} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 131 131 131 226 226 226 T T T 1 020-039 131 131 131 226 226 226 F F F 2 040-059 117 117 117 207 207 207 T T T 3 060-079 131 131 131 226 226 226 T T T 4 080-099 131 131 131 226 226 226 F F F 5 100-119 130 130 130 225 225 225 T T T 6 120-139 130 130 130 225 225 225 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 130 130 130 225 225 225 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 126 126 126 221 221 221 T T T 13 260-279 130 130 130 225 225 225 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 131 131 131 226 226 226 T T T 18 360-379 130 130 130 225 225 225 T T T 19 380-399 131 131 131 226 226 226 F F F 20 400-419 131 131 131 226 226 226 T T T 21 420-439 130 130 130 225 225 225 T T T 22 440-459 131 131 131 226 226 226 F F F 23 460-479 126 126 126 221 221 221 T T T 24 480-499 126 126 126 221 221 221 T T T 25 500-519 128 128 128 222 222 222 T T T 26 520-539 131 131 131 226 226 226 F F F 27 540-559 128 128 128 223 223 223 T T T 28 560-579 131 131 131 226 226 226 F F F 29 580-599 128 128 128 222 222 222 T T T Max gen: 131, Max flush: 226 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 131 131 131 226 226 226 F F F 2 040-059 132 132 132 227 227 227 F F F 3 060-079 132 132 132 227 227 227 F F F 4 080-099 131 131 131 226 226 226 F F F 5 100-119 132 132 132 227 227 227 F F F 6 120-139 130 130 130 225 225 225 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 130 130 130 225 225 225 F F F 9 180-199 112 112 112 203 203 203 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 126 126 126 221 221 221 F F F 12 240-259 132 132 132 227 227 227 F F F 13 260-279 130 130 130 225 225 225 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 128 128 128 222 222 222 F F F 17 340-359 132 132 132 227 227 227 F F F 18 360-379 132 132 132 227 227 227 F F F 19 380-399 131 131 131 226 226 226 F F F 20 400-419 132 132 132 227 227 227 F F F 21 420-439 132 132 132 227 227 227 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 131 131 131 226 226 226 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 131 131 131 226 226 226 F F F 29 580-599 132 132 132 227 227 227 F F F Max gen: 132, Max flush: 227 Begin loop 68 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.677984059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:09.720111048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.720506984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.751189777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.751850983Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.751886412Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.751900381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.75191514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} {"msg":"Crucible 99548608-fcfe-4d95-bd05-0b8e103469e7 has session id: 52bf7bbb-647c-4a8b-b34c-7566522f9d04","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.752422512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Upstairs opts: Upstairs UUID: 99548608-fcfe-4d95-bd05-0b8e103469e7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.75247534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Crucible stats registered with UUID: 99548608-fcfe-4d95-bd05-0b8e103469e7","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.75249919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782276844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782309573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} {"msg":"99548608-fcfe-4d95-bd05-0b8e103469e7 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782379381Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.78240264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782435789Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782455808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782473268Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782499047Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782527895Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782559364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782591483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782624072Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_connection connected from Ok(127.0.0.1:54928)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782657611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"io task","client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_connection connected from Ok(127.0.0.1:63894)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.78269294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"io task","client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_connection connected from Ok(127.0.0.1:40822)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.782844104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"io task","client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.783444552Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.783484891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.78351668Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.783549829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.783579388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.783604016Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784115368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784157487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784192865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784698477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784736046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784762805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784777144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784791524Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784804953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784817313Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784830652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[0]R flush_numbers[0..12]: [227, 226, 227, 227, 226, 227, 225, 226, 225, 203, 226, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784844542Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[0]R generation[0..12]: [132, 131, 132, 132, 131, 132, 130, 131, 130, 112, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784858741Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784876541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[1]R flush_numbers[0..12]: [227, 226, 227, 227, 226, 227, 225, 226, 225, 203, 226, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.78489089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[1]R generation[0..12]: [132, 131, 132, 132, 131, 132, 130, 131, 130, 112, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.78490614Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784928309Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {Wait for a query_work_queue command to finish before sending IO "msg":"[2]R flush_numbers[0..12]: [227, 226, 227, 227, 226, 227, 225, 226, 225, 203, 226, 221]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784965778Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[2]R generation[0..12]: [132, 131, 132, 132, 131, 132, 130, 131, 130, 112, 131, 126]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.784988887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785003756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Max found gen is 133","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785016965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Generation requested: 136 >= found:133","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785030125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Next flush: 228","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785043095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785056464Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"":"downstairs","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785069704Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785081763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"0","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785094973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"1","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785108332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"client":"2","":"downstairs","session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785124412Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785137201Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"99548608-fcfe-4d95-bd05-0b8e103469e7 is now active with session: 52bf7bbb-647c-4a8b-b34c-7566522f9d04","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.785150051Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:09.78516219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726,"session_id":"52bf7bbb-647c-4a8b-b34c-7566522f9d04"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:08:09.78517466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 163 len 40960 data: 9 9 11 11 10 9 7 7 7 7 02/30 Write block 390 len 32768 data: 8 10 12 11 14 16 16 16 03/30 Write block 577 len 8192 data: 7 8 04/30 Read block 515 len 36864 05/30 Read block 219 len 40960 06/30 Write block 367 len 12288 data: 11 10 14 07/30 Write block 345 len 32768 data: 11 10 10 13 13 9 7 7 08/30 Read block 317 len 28672 09/30 Write block 226 len 28672 data: 7 8 7 8 8 9 7 10/30 Read block 582 len 8192 11/30 Read block 472 len 32768 12/30 Flush 13/30 Flush 14/30 Write block 351 len 8192 data: 8 8 15/30 Read block 190 len 40960 16/30 Write block 321 len 4096 data: 10 17/30 Flush 18/30 Write block 187 len 40960 data: 12 10 10 8 7 7 9 11 8 9 19/30 Read block 406 len 8192 20/30 Read block 274 len 32768 21/30 Write block 258 len 32768 data: 10 8 8 7 7 6 7 9 22/30 Write block 38 len 16384 data: 15 13 12 12 23/30 Write block 29 len 8192 data: 14 15 24/30 Read block 248 len 4096 25/30 Write block 429 len 28672 data: 10 9 8 10 10 10 9 26/30 Write block 185 len 40960 data: 11 13 13 11 11 9 8 8 10 12 27/30 Read block 68 len 12288 28/30 Write block 336 len 40960 data: 15 15 16 11 10 13 15 15 14 12 29/30 Write block 173 len 16384 data: 6 6 7 6 30/30 Read block 335 len 32768 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 10 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Read 8 Done Done Done false 1020 Acked Write 8 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 1 Done Done Done false 1024 Acked Write 7 Done Done Done false 1025 Acked Write 10 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 4 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 0 0 2 Done 11 13 13 37 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:09.808885824Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1726} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 131 131 131 226 226 226 T T T 2 040-059 132 132 132 227 227 227 T T T 3 060-079 132 132 132 227 227 227 F F F 4 080-099 131 131 131 226 226 226 F F F 5 100-119 132 132 132 227 227 227 F F F 6 120-139 130 130 130 225 225 225 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 136 136 136 228 228 228 T T T 9 180-199 112 112 112 203 203 203 T T T 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 132 132 132 227 227 227 T T T 13 260-279 130 130 130 225 225 225 T T T 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 136 136 136 230 230 230 T T T 17 340-359 136 136 136 230 230 230 T T T 18 360-379 136 136 136 228 228 228 F F F 19 380-399 136 136 136 228 228 228 F F F 20 400-419 132 132 132 227 227 227 F F F 21 420-439 132 132 132 227 227 227 T T T 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 131 131 131 226 226 226 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 136 136 136 228 228 228 F F F 29 580-599 132 132 132 227 227 227 F F F Max gen: 136, Max flush: 230 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 137 137 137 231 231 231 F F F 2 040-059 137 137 137 231 231 231 F F F 3 060-079 132 132 132 227 227 227 F F F 4 080-099 131 131 131 226 226 226 F F F 5 100-119 132 132 132 227 227 227 F F F 6 120-139 130 130 130 225 225 225 F F F 7 140-159 131 131 131 226 226 226 F F F 8 160-179 137 137 137 231 231 231 F F F 9 180-199 137 137 137 231 231 231 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 137 137 137 231 231 231 F F F 17 340-359 137 137 137 231 231 231 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 136 136 136 228 228 228 F F F 20 400-419 132 132 132 227 227 227 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 131 131 131 226 226 226 F F F 27 540-559 132 132 132 227 227 227 F F F 28 560-579 136 136 136 228 228 228 F F F 29 580-599 132 132 132 227 227 227 F F F Max gen: 137, Max flush: 231 Begin loop 69 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.203530882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:14.245660572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.246042208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.277021189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.27755269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.277597269Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.277615528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.277633737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} {"msg":"Crucible 88777ad9-2660-41fc-aefc-24c345b00c0c has session id: 09014e7f-dbae-4057-a939-8e8ae1053327","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.278048002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Upstairs opts: Upstairs UUID: 88777ad9-2660-41fc-aefc-24c345b00c0c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.278077121Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Crucible stats registered with UUID: 88777ad9-2660-41fc-aefc-24c345b00c0c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.278091531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.307886375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.307918334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} {"msg":"88777ad9-2660-41fc-aefc-24c345b00c0c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.307956233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.307978542Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.30802854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308065259Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308102318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308135766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308169415Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308195584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308220323Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308252472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_connection connected from Ok(127.0.0.1:58087)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308384567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"io task","client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_connection connected from Ok(127.0.0.1:63714)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308418846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"io task","client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_connection connected from Ok(127.0.0.1:61112)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.308447935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"io task","client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309041333Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309082512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309114561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.30914803Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309185198Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309210818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309735239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309775807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.309810146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310327067Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310366446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310393645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310423114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310438763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310452263Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310464652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310478102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[0]R flush_numbers[0..12]: [227, 231, 231, 227, 226, 227, 225, 226, 231, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310494831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[0]R generation[0..12]: [132, 137, 137, 132, 131, 132, 130, 131, 137, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.31052149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310555119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[1]R flush_numbers[0..12]: [227, 231, 231, 227, 226, 227, 225, 226, 231, 231, 226, 228]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:08:14.310586108Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[1]R generation[0..12]: [132, 137, 137, 132, 131, 132, 130, 131, 137, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310624816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310643466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[2]R flush_numbers[0..12]: [227, 231, 231, 227, 226, 227, 225, 226, 231, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310658775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[2]R generation[0..12]: [132, 137, 137, 132, 131, 132, 130, 131, 137, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310672955Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310686964Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310700114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310713753Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Next flush: 232","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310727153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310740112Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"":"downstairs","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:08:14.310753712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310766071Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"0","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310779801Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"1","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.31079645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"client":"2","":"downstairs","session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.3108104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310823489Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"88777ad9-2660-41fc-aefc-24c345b00c0c is now active with session: 09014e7f-dbae-4057-a939-8e8ae1053327","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310836519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310848448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734,"session_id":"09014e7f-dbae-4057-a939-8e8ae1053327"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:14.310860678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 96 len 24576 data: 11 9 5 4 5 7 02/30 Read block 366 len 24576 03/30 Write block 327 len 24576 data: 9 10 10 13 16 15 04/30 Read block 15 len 24576 05/30 Write block 544 len 28672 data: 7 8 13 15 19 17 16 06/30 Flush 07/30 Write block 29 len 32768 data: 15 16 16 12 11 14 11 11 08/30 Read block 275 len 20480 09/30 Read block 429 len 8192 10/30 Read block 460 len 20480 11/30 Read block 570 len 24576 12/30 Read block 432 len 28672 13/30 Read block 310 len 32768 14/30 Write block 132 len 36864 data: 16 11 14 12 12 11 8 9 8 15/30 Flush 16/30 Write block 34 len 28672 data: 15 12 12 14 16 14 13 17/30 Flush 18/30 Flush 19/30 Flush 20/30 Write block 566 len 8192 data: 10 13 21/30 Read block 550 len 36864 22/30 Read block 128 len 28672 23/30 Read block 80 len 28672 24/30 Read block 338 len 16384 25/30 Read block 590 len 28672 26/30 Read block 19 len 40960 27/30 Read block 494 len 28672 28/30 Read block 499 len 16384 29/30 Read block 174 len 24576 30/30 Read block 582 len 20480 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 2 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 7 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 10 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 5 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 11 11 11 33 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:14.337089451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1734} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 138 138 138 234 234 234 F F F 2 040-059 138 138 138 234 234 234 F F F 3 060-079 132 132 132 227 227 227 F F F 4 080-099 138 138 138 232 232 232 F F F 5 100-119 138 138 138 232 232 232 F F F 6 120-139 138 138 138 233 233 233 F F F 7 140-159 138 138 138 233 233 233 F F F 8 160-179 137 137 137 231 231 231 F F F 9 180-199 137 137 137 231 231 231 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 138 138 138 232 232 232 F F F 17 340-359 137 137 137 231 231 231 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 136 136 136 228 228 228 F F F 20 400-419 132 132 132 227 227 227 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 131 131 131 226 226 226 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 136 136 136 228 228 228 T T T 29 580-599 132 132 132 227 227 227 F F F Max gen: 138, Max flush: 234 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 138 138 138 234 234 234 F F F 2 040-059 138 138 138 234 234 234 F F F 3 060-079 132 132 132 227 227 227 F F F 4 080-099 138 138 138 232 232 232 F F F 5 100-119 138 138 138 232 232 232 F F F 6 120-139 138 138 138 233 233 233 F F F 7 140-159 138 138 138 233 233 233 F F F 8 160-179 137 137 137 231 231 231 F F F 9 180-199 137 137 137 231 231 231 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 130 130 130 225 225 225 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 138 138 138 232 232 232 F F F 17 340-359 137 137 137 231 231 231 F F F 18 360-379 136 136 136 228 228 228 F F F 19 380-399 136 136 136 228 228 228 F F F 20 400-419 132 132 132 227 227 227 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 F F F 26 520-539 131 131 131 226 226 226 F F F 27 540-559 138 138 138 232 232 232 F F F 28 560-579 139 139 139 235 235 235 F F F 29 580-599 132 132 132 227 227 227 F F F Max gen: 139, Max flush: 235 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.63059122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:18.672773207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.673143634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.703686281Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.704224972Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.70426847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.70428585Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.704303829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} {"msg":"Crucible e58b40af-015d-47b5-935f-c1305659f36f has session id: 71fec3a8-796a-44fd-b5de-cc4c6471041a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.704721014Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Upstairs opts: Upstairs UUID: e58b40af-015d-47b5-935f-c1305659f36f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.704762643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Crucible stats registered with UUID: e58b40af-015d-47b5-935f-c1305659f36f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.704788562Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734566657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734598255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} {"msg":"e58b40af-015d-47b5-935f-c1305659f36f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734633454Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734652004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734678723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734700592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734718981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73473551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73475257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734769219Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734801398Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734834747Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_connection connected from Ok(127.0.0.1:57976)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.734975192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"io task","client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_connection connected from Ok(127.0.0.1:37153)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73502821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"io task","client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_connection connected from Ok(127.0.0.1:54125)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.735154435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"io task","client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.735769363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.735798382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.735960566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.735980896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.736047083Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.736071502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.736459168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.736732058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.736760668Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737159073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737188752Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737204481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737217411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73723128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73724486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737257599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737271249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[0]R flush_numbers[0..12]: [227, 234, 234, 227, 232, 232, 233, 233, 231, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737285129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[0]R generation[0..12]: [132, 138, 138, 132, 138, 138, 138, 138, 137, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737299118Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737313277Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[1]R flush_numbers[0..12]: [227, 234, 234, 227, 232, 232, 233, 233, 231, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737327117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[1]R generation[0..12]: [132, 138, 138, 132, 138, 138, 138, 138, 137, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737344246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737358896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[2]R flush_numbers[0..12]: [227, 234, 234, 227, 232, 232, 233, 233, 231, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737372785Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[2]R generation[0..12]: [132, 138, 138, 132, 138, 138, 138, 138, 137, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737386695Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737400744Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737413894Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737427363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Next flush: 236","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737440563Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:08:18.737453692Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"":"downstairs","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737472132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737484201Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"0","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737497551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"1","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73751068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"client":"2","":"downstairs","session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.73752403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737537989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"e58b40af-015d-47b5-935f-c1305659f36f is now active with session: 71fec3a8-796a-44fd-b5de-cc4c6471041a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737551089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737563308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742,"session_id":"71fec3a8-796a-44fd-b5de-cc4c6471041a"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:18.737575748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 293 len 20480 data: 8 6 9 11 11 02/30 Read block 122 len 12288 03/30 Read block 380 len 20480 04/30 Read block 535 len 36864 05/30 Write block 101 len 40960 data: 8 11 11 12 14 15 12 13 12 13 06/30 Write block 395 len 32768 data: 17 17 17 17 16 14 11 9 07/30 Read block 201 len 12288 08/30 Read block 367 len 24576 09/30 Write block 537 len 40960 data: 9 10 7 7 7 6 5 8 9 14 10/30 Write block 400 len 12288 data: 15 12 10 11/30 Write block 572 len 40960 data: 10 9 7 6 5 8 9 6 6 5 12/30 Write block 166 len 24576 data: 12 11 10 8 8 8 13/30 Flush 14/30 Write block 589 len 16384 data: 16 15 11 11 15/30 Write block 376 len 12288 data: 13 13 13 16/30 Flush 17/30 Write block 563 len 36864 data: 6 7 11 11 14 13 14 14 14 18/30 Read block 285 len 16384 19/30 Write block 31 len 32768 data: 17 13 12 16 13 13 15 17 20/30 Read block 270 len 24576 21/30 Write block 94 len 8192 data: 15 12 22/30 Write block 137 len 28672 data: 12 9 10 9 5 7 10 23/30 Flush 24/30 Write block 500 len 4096 data: 14 25/30 Read block 358 len 8192 26/30 Read block 340 len 32768 27/30 Write block 320 len 40960 data: 10 11 11 9 7 8 8 10 11 11 28/30 Read block 41 len 36864 29/30 Write block 399 len 4096 data: 17 30/30 Read block 503 len 40960 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 1 Done Done Done false 1024 Acked Read 2 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 9 Done Done Done false 1028 Acked Write 1 Done Done Done false 1029 Acked Read 10 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 6 7 7 20 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:18.763804181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1742} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 138 138 138 234 234 234 F F F 3 060-079 132 132 132 227 227 227 F F F 4 080-099 140 140 140 238 238 238 F F F 5 100-119 140 140 140 236 236 236 F F F 6 120-139 140 140 140 238 238 238 F F F 7 140-159 140 140 140 238 238 238 F F F 8 160-179 140 140 140 236 236 236 F F F 9 180-199 137 137 137 231 231 231 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 140 140 140 236 236 236 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 138 138 138 232 232 232 T T T 17 340-359 137 137 137 231 231 231 F F F 18 360-379 140 140 140 237 237 237 F F F 19 380-399 140 140 140 236 236 236 T T T 20 400-419 140 140 140 236 236 236 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 132 132 132 227 227 227 T T T 26 520-539 140 140 140 236 236 236 F F F 27 540-559 140 140 140 236 236 236 F F F 28 560-579 140 140 140 238 238 238 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 140, Max flush: 238 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 140 140 140 238 238 238 F F F 2 040-059 138 138 138 234 234 234 F F F 3 060-079 132 132 132 227 227 227 F F F 4 080-099 140 140 140 238 238 238 F F F 5 100-119 140 140 140 236 236 236 F F F 6 120-139 140 140 140 238 238 238 F F F 7 140-159 140 140 140 238 238 238 F F F 8 160-179 140 140 140 236 236 236 F F F 9 180-199 137 137 137 231 231 231 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 140 140 140 236 236 236 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 141 141 141 239 239 239 F F F 17 340-359 137 137 137 231 231 231 F F F 18 360-379 140 140 140 237 237 237 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 140 140 140 236 236 236 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 F F F 24 480-499 132 132 132 227 227 227 F F F 25 500-519 141 141 141 239 239 239 F F F 26 520-539 140 140 140 236 236 236 F F F 27 540-559 140 140 140 236 236 236 F F F 28 560-579 140 140 140 238 238 238 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 141, Max flush: 239 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.080365428Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.123738702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.124491565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.124528334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.124547023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.124561973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} {"msg":"Crucible a30de8c0-08f6-4c44-8552-0b36b0125f03 has session id: 758927da-d0ee-4646-815d-ccd4fc7e8171","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.124974818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Upstairs opts: Upstairs UUID: a30de8c0-08f6-4c44-8552-0b36b0125f03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.125017256Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Crucible stats registered with UUID: a30de8c0-08f6-4c44-8552-0b36b0125f03","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.125045025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.154993274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155068681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} {"msg":"a30de8c0-08f6-4c44-8552-0b36b0125f03 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.15509971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155130649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155178597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155217246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155244595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155267364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155303793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155337842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.15537347Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155408339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_connection connected from Ok(127.0.0.1:63089)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155497596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"io task","client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_connection connected from Ok(127.0.0.1:41450)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.155623891Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"io task","client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_connection connected from Ok(127.0.0.1:41462)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.15565882Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"io task","client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156301597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156331136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156346805Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156361015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156376134Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156402123Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.156977712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157015311Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.15704459Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157528352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157555451Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157570061Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157637879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157657368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157671367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157683817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157697426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[0]R flush_numbers[0..12]: [227, 238, 234, 227, 238, 236, 238, 238, 236, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157711536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[0]R generation[0..12]: [132, 140, 138, 132, 140, 140, 140, 140, 140, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157725445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157739715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[1]R flush_numbers[0..12]: [227, 238, 234, 227, 238, 236, 238, 238, 236, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157753704Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[1]R generation[0..12]: [132, 140, 138, 132, 140, 140, 140, 140, 140, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157767504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157781523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[2]R flush_numbers[0..12]: [227, 238, 234, 227, 238, 236, 238, 238, 236, 231, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157798453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[2]R generation[0..12]: [132, 140, 138, 132, 140, 140, 140, 140, 140, 137, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157812592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157826892Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157840311Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157853861Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.15786713Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"All extents match","v":0,"name":"crucible","level":30,Wait for a query_work_queue command to finish before sending IO "time":"2024-11-07T14:08:23.15788032Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"":"downstairs","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157902539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157914958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"0","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157928678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"1","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157942257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"client":"2","":"downstairs","session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157955827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157968316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"a30de8c0-08f6-4c44-8552-0b36b0125f03 is now active with session: 758927da-d0ee-4646-815d-ccd4fc7e8171","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157981206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.157993366Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750,"session_id":"758927da-d0ee-4646-815d-ccd4fc7e8171"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:23.158019475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 567 len 32768 data: 15 14 15 15 15 11 10 8 02/30 Write block 359 len 32768 data: 11 14 13 13 14 13 10 10 03/30 Write block 328 len 24576 data: 12 12 14 17 16 16 04/30 Read block 16 len 8192 05/30 Write block 345 len 24576 data: 13 11 11 14 14 10 06/30 Write block 60 len 28672 data: 12 9 11 10 8 8 9 07/30 Write block 29 len 4096 data: 16 08/30 Read block 199 len 32768 09/30 Flush 10/30 Flush 11/30 Write block 111 len 40960 data: 13 14 15 13 14 14 14 13 14 15 12/30 Read block 100 len 28672 13/30 Write block 139 len 36864 data: 11 10 6 8 11 13 17 18 19 14/30 Read block 221 len 16384 15/30 Write block 499 len 28672 data: 15 15 12 13 13 13 10 16/30 Read block 366 len 12288 17/30 Write block 464 len 36864 data: 12 13 12 12 10 10 10 9 8 18/30 Read block 341 len 8192 19/30 Read block 36 len 8192 20/30 Write block 330 len 12288 data: 15 18 17 21/30 Read block 184 len 20480 22/30 Read block 455 len 8192 23/30 Write block 471 len 28672 data: 10 9 11 14 12 8 7 24/30 Read block 40 len 40960 25/30 Write block 540 len 20480 data: 8 8 7 6 9 26/30 Read block 476 len 4096 27/30 Read block 549 len 8192 28/30 Write block 283 len 24576 data: 19 16 16 17 17 16 29/30 Read block 165 len 36864 30/30 Write block 178 len 40960 data: 6 8 9 9 9 10 9 12 14 14 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:20 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1010 Acked Write 10 Done Done Done false 1011 Acked Read 7 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 4 Done Done Done false 1014 Acked Write 7 Done Done Done false 1015 Acked Read 3 Done Done Done false 1016 Acked Write 9 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 3 Done Done Done false 1020 Acked Read 5 Done Done Done false 1021 Acked Read 2 Done Done Done false 1022 Acked Write 7 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Write 5 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 6 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 19 19 19 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1009 1009 1009 Downstairs last five completed: 1009 1008 1007 1006 1005 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:23.181432949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1750} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 142 142 142 240 240 240 F F F 2 040-059 138 138 138 234 234 234 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 140 140 140 238 238 238 F F F 5 100-119 140 140 140 236 236 236 T T T 6 120-139 140 140 140 238 238 238 T T T 7 140-159 140 140 140 238 238 238 T T T 8 160-179 140 140 140 236 236 236 T T T 9 180-199 137 137 137 231 231 231 T T T 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 140 140 140 236 236 236 T T T 15 300-319 130 130 130 225 225 225 F F F 16 320-339 142 142 142 240 240 240 T T T 17 340-359 142 142 142 240 240 240 F F F 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 140 140 140 236 236 236 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 132 132 132 227 227 227 T T T 24 480-499 132 132 132 227 227 227 T T T 25 500-519 141 141 141 239 239 239 T T T 26 520-539 140 140 140 236 236 236 F F F 27 540-559 140 140 140 236 236 236 T T T 28 560-579 142 142 142 240 240 240 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 142, Max flush: 240 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 142 142 142 240 240 240 F F F 2 040-059 138 138 138 234 234 234 F F F 3 060-079 142 142 142 240 240 240 F F F 4 080-099 140 140 140 238 238 238 F F F 5 100-119 143 143 143 241 241 241 F F F 6 120-139 143 143 143 241 241 241 F F F 7 140-159 143 143 143 241 241 241 F F F 8 160-179 143 143 143 241 241 241 F F F 9 180-199 143 143 143 241 241 241 F F F 10 200-219 131 131 131 226 226 226 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 143 143 143 241 241 241 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 143 143 143 241 241 241 F F F 17 340-359 142 142 142 240 240 240 F F F 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 140 140 140 236 236 236 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 143 143 143 241 241 241 F F F 25 500-519 143 143 143 241 241 241 F F F 26 520-539 140 140 140 236 236 236 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 142 142 142 240 240 240 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 143, Max flush: 241 Begin loop 72 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.577437719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.62062147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621338944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621376683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621395232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621410222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} {"msg":"Crucible cf55418e-223f-419a-88d1-d41f1ac747fa has session id: 0fddefaa-07f4-4327-8e1d-c5a57cee02b6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621837036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Upstairs opts: Upstairs UUID: cf55418e-223f-419a-88d1-d41f1ac747fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621878195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Crucible stats registered with UUID: cf55418e-223f-419a-88d1-d41f1ac747fa","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.621897314Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.65330958Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653345369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} {"msg":"cf55418e-223f-419a-88d1-d41f1ac747fa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653378248Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653396417Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653424846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653443705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653460985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653480984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653517003Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653544202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653569021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.65359194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:57028)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653618659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"io task","client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:35649)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653801392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"io task","client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_connection connected from Ok(127.0.0.1:34573)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.653841101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"io task","client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.654374932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.65443084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.654467008Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.654493677Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.654652362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.654673431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.65526636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655298868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655315138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655832239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655858748Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655873387Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655935305Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655952355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655966124Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655978494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.655992093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[0]R flush_numbers[0..12]: [227, 240, 234, 240, 238, 241, 241, 241, 241, 241, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656006333Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[0]R generation[0..12]: [132, 142, 138, 142, 140, 143, 143, 143, 143, 143, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656020482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656034812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[1]R flush_numbers[0..12]: [227, 240, 234, 240, 238, 241, 241, 241, 241, 241, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656048671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[1]R generation[0..12]: [132, 142, 138, 142, 140, 143, 143, 143, 143, 143, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656062551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.65607672Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[2]R flush_numbers[0..12]: [227, 240, 234, 240, 238, 241, 241, 241, 241, 241, 226, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.6560937Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[2]R generation[0..12]: [132, 142, 138, 142, 140, 143, 143, 143, 143, 143, 131, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656107929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656122098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656135308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656148598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Next flush: 242","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656161777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656174917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":Wait for a query_work_queue command to finish before sending IO 1758,"":"downstairs","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656196666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656209226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"0","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656231975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"1","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656246554Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"client":"2","":"downstairs","session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656260084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656272523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"cf55418e-223f-419a-88d1-d41f1ac747fa is now active with session: 0fddefaa-07f4-4327-8e1d-c5a57cee02b6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656285433Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656297442Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758,"session_id":"0fddefaa-07f4-4327-8e1d-c5a57cee02b6"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:27.656309812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 181 len 36864 data: 10 10 11 10 13 15 15 12 12 02/30 Flush 03/30 Write block 109 len 24576 data: 13 14 14 15 16 14 04/30 Write block 174 len 40960 data: 7 8 7 7 7 9 10 11 11 12 05/30 Write block 216 len 16384 data: 14 14 12 11 06/30 Read block 510 len 8192 07/30 Read block 95 len 8192 08/30 Flush 09/30 Write block 293 len 24576 data: 9 7 10 12 12 13 10/30 Write block 324 len 12288 data: 8 9 9 11/30 Read block 573 len 12288 12/30 Read block 129 len 4096 13/30 Flush 14/30 Write block 524 len 24576 data: 11 9 10 8 7 8 15/30 Write block 357 len 8192 data: 12 11 16/30 Read block 361 len 28672 17/30 Write block 162 len 20480 data: 10 10 10 12 13 18/30 Read block 359 len 24576 19/30 Write block 518 len 16384 data: 11 9 10 12 20/30 Write block 31 len 16384 data: 18 14 13 17 21/30 Read block 408 len 28672 22/30 Read block 121 len 36864 23/30 Write block 55 len 32768 data: 9 10 10 12 11 13 10 12 24/30 Read block 223 len 12288 25/30 Read block 239 len 24576 26/30 Write block 321 len 24576 data: 12 12 10 9 10 10 27/30 Write block 345 len 40960 data: 14 12 12 15 15 11 9 9 7 10 28/30 Read block 383 len 24576 29/30 Write block 21 len 8192 data: 12 15 30/30 Write block 486 len 16384 data: 11 11 10 9 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 6 Done Done Done false 1014 Acked Write 2 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 5 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 4 Done Done Done false 1019 Acked Write 4 Done Done Done false 1020 Acked Read 7 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Write 10 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 1 2 5 Done 15 16 15 46 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:27.681485203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1758} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 142 142 142 240 240 240 T T T 2 040-059 138 138 138 234 234 234 T T T 3 060-079 142 142 142 240 240 240 T T T 4 080-099 140 140 140 238 238 238 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 143 143 143 241 241 241 F F F 7 140-159 143 143 143 241 241 241 F F F 8 160-179 144 144 144 243 243 243 T T T 9 180-199 144 144 144 243 243 243 F F F 10 200-219 144 144 144 243 243 243 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 144 144 144 244 244 244 T T T 17 340-359 142 142 142 240 240 240 T T T 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 140 140 140 236 236 236 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 143 143 143 241 241 241 T T T 25 500-519 143 143 143 241 241 241 T T T 26 520-539 140 140 140 236 236 236 T T T 27 540-559 143 143 143 241 241 241 F F F 28 560-579 142 142 142 240 240 240 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 144, Max flush: 244 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 145 145 145 245 245 245 F F F 4 080-099 140 140 140 238 238 238 F F F 5 100-119 144 144 144 243 243 243 F F F 6 120-139 143 143 143 241 241 241 F F F 7 140-159 143 143 143 241 241 241 F F F 8 160-179 145 145 145 245 245 245 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 144 144 144 243 243 243 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 137 137 137 231 231 231 F F F 13 260-279 137 137 137 231 231 231 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 145 145 145 245 245 245 F F F 17 340-359 145 145 145 245 245 245 F F F 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 140 140 140 236 236 236 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 131 131 131 226 226 226 F F F 23 460-479 143 143 143 241 241 241 F F F 24 480-499 145 145 145 245 245 245 F F F 25 500-519 145 145 145 245 245 245 F F F 26 520-539 145 145 145 245 245 245 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 142 142 142 240 240 240 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 145, Max flush: 245 Begin loop 73 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.047921209Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:32.09057302Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.090951426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.122266156Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.122866714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.122902363Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.122916762Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.122931292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} {"msg":"Crucible fed9340e-8919-4894-8d89-08df57375887 has session id: 433e0721-3807-4733-969b-54848de64480","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.123351147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Upstairs opts: Upstairs UUID: fed9340e-8919-4894-8d89-08df57375887, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.123392485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Crucible stats registered with UUID: fed9340e-8919-4894-8d89-08df57375887","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.123417204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153077473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153110152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} {"msg":"fed9340e-8919-4894-8d89-08df57375887 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15316074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15318139Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153208529Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153226378Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153243647Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153260237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153280526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153297386Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153314775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153331434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_connection connected from Ok(127.0.0.1:43078)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153477729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"io task","client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_connection connected from Ok(127.0.0.1:35551)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153552846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"io task","client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_connection connected from Ok(127.0.0.1:49603)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.153590595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"io task","client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154205323Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154245641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15427744Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154307059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154334638Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154349808Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154881098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154912607Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.154945616Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155436838Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155478207Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155509336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155572213Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155603432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155620012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155633321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155647241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[0]R flush_numbers[0..12]: [227, 245, 245, 245, 238, 243, 241, 241, 245, 243, 243, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15566151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[0]R generation[0..12]: [132, 145, 145, 145, 140, 144, 143, 143, 145, 144, 144, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15567584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155690119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[1]R flush_numbers[0..12]: [227, 245, 245, 245, 238, 243, 241, 241, 245, 243, 243, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155704188Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[1]R generation[0..12]: [132, 145, 145, 145, 140, 144, 143, 143, 145, 144, 144, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155718168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155736767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[2]R flush_numbers[0..12]: [227, 245, 245, 245, 238, 243, 241, 241, 245, 243, 243, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155751537Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[2]R generation[0..12]: [132, 145, 145, 145, 140, 144, 143, 143, 145, 144, 144, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155765336Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155779436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":Wait for a query_work_queue command to finish before sending IO "2024-11-07T14:08:32.155792645Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155814815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155827974Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155840984Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"":"downstairs","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155854203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155866293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"0","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155879622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"1","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155892932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"client":"2","":"downstairs","session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155920721Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15593608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"fed9340e-8919-4894-8d89-08df57375887 is now active with session: 433e0721-3807-4733-969b-54848de64480","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.15595027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155962719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766,"session_id":"433e0721-3807-4733-969b-54848de64480"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:32.155977229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 145 len 12288 data: 18 19 20 02/30 Write block 531 len 12288 data: 13 12 10 03/30 Write block 216 len 4096 data: 15 04/30 Read block 236 len 20480 05/30 Read block 308 len 8192 06/30 Write block 206 len 20480 data: 15 12 11 11 9 07/30 Read block 34 len 28672 08/30 Read block 335 len 36864 09/30 Read block 211 len 8192 10/30 Write block 210 len 16384 data: 10 8 6 9 11/30 Read block 52 len 32768 12/30 Flush 13/30 Write block 408 len 24576 data: 9 12 13 12 12 12 14/30 Write block 254 len 4096 data: 14 15/30 Read block 295 len 4096 16/30 Flush 17/30 Write block 459 len 32768 data: 14 11 15 15 15 13 14 13 18/30 Read block 400 len 32768 19/30 Flush 20/30 Write block 118 len 4096 data: 14 21/30 Flush 22/30 Flush 23/30 Flush 24/30 Write block 254 len 28672 data: 15 17 17 14 11 9 9 25/30 Read block 173 len 24576 26/30 Read block 293 len 32768 27/30 Read block 594 len 20480 28/30 Write block 85 len 28672 data: 10 13 17 18 19 20 16 29/30 Write block 168 len 4096 data: 11 30/30 Write block 95 len 16384 data: 13 12 10 6 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Read 8 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 7 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 3 9 Done 4 4 4 12 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:32.181071483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1766} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 145 145 145 245 245 245 F F F 4 080-099 140 140 140 238 238 238 T T T 5 100-119 146 146 146 249 249 249 F F F 6 120-139 143 143 143 241 241 241 F F F 7 140-159 146 146 146 246 246 246 F F F 8 160-179 145 145 145 245 245 245 T T T 9 180-199 144 144 144 243 243 243 F F F 10 200-219 146 146 146 246 246 246 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 146 146 146 247 247 247 T T T 13 260-279 137 137 137 231 231 231 T T T 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 145 145 145 245 245 245 F F F 17 340-359 145 145 145 245 245 245 F F F 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 146 146 146 248 248 248 F F F 23 460-479 146 146 146 248 248 248 F F F 24 480-499 145 145 145 245 245 245 F F F 25 500-519 145 145 145 245 245 245 F F F 26 520-539 146 146 146 246 246 246 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 142 142 142 240 240 240 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 146, Max flush: 249 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 F F F 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 145 145 145 245 245 245 F F F 4 080-099 147 147 147 250 250 250 F F F 5 100-119 146 146 146 249 249 249 F F F 6 120-139 143 143 143 241 241 241 F F F 7 140-159 146 146 146 246 246 246 F F F 8 160-179 147 147 147 250 250 250 F F F 9 180-199 144 144 144 243 243 243 F F F 10 200-219 146 146 146 246 246 246 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 147 147 147 250 250 250 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 145 145 145 245 245 245 F F F 17 340-359 145 145 145 245 245 245 F F F 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 146 146 146 248 248 248 F F F 23 460-479 146 146 146 248 248 248 F F F 24 480-499 145 145 145 245 245 245 F F F 25 500-519 145 145 145 245 245 245 F F F 26 520-539 146 146 146 246 246 246 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 142 142 142 240 240 240 F F F 29 580-599 140 140 140 237 237 237 F F F Max gen: 147, Max flush: 250 Begin loop 74 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.513664711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:36.555381975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.555763521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.586915307Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.587450817Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.587492856Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.587515285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.587534274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} {"msg":"Crucible 7d4fe1e3-12b4-489d-9964-45c836d94650 has session id: fba78f9e-14ee-4ba0-9182-5909e856d118","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.5879201Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Upstairs opts: Upstairs UUID: 7d4fe1e3-12b4-489d-9964-45c836d94650, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.587969439Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Crucible stats registered with UUID: 7d4fe1e3-12b4-489d-9964-45c836d94650","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.587998338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619034907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619073716Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} {"msg":"7d4fe1e3-12b4-489d-9964-45c836d94650 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619112374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619132734Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619177352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619214661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619250749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619281798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619311947Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619341376Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619375995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619406244Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_connection connected from Ok(127.0.0.1:44390)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619432853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"io task","client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_connection connected from Ok(127.0.0.1:39682)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.61952073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"io task","client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_connection connected from Ok(127.0.0.1:62363)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.619625026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"io task","client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620214555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620243514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620259253Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620273182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620286642Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620299912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620817743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620848792Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.620880551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621432291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621466349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621483249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621496318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621510478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621524337Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621536837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621550346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[0]R flush_numbers[0..12]: [227, 245, 245, 245, 250, 249, 241, 246, 250, 243, 246, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621564396Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[0]R generation[0..12]: [132, 145, 145, 145, 147, 146, 143, 146, 147, 144, 146, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621578525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621592665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[1]R flush_numbers[0..12]: [227, 245, 245, 245, 250, 249, 241, 246, 250, 243, 246, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621606634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[1]R generation[0..12]: [132, 145, 145, 145, 147, 146, 143, 146, 147, 144, 146, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621620454Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621634423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[2]R flush_numbers[0..12]: [227, 245, 245, 245, 250, 249, 241, 246, 250, 243, 246, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621648373Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[2]R generation[0..12]: [132, 145, 145, 145, 147, 146, 143, 146, 147, 144, 146, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621664012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621687171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621700571Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621713881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.62172696Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621740049Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"":"downstairs","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621753499Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621765779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"0","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621779148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"1","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621792578Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"client":"2","":"downstairs","session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621805897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621818237Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"7d4fe1e3-12b4-489d-9964-45c836d94650 is now active with session: fba78f9e-14ee-4ba0-9182-5909e856d118","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621831076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621843126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774,"session_id":"fba78f9e-14ee-4ba0-9182-5909e856d118"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:36.621855445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 331 len 20480 data: 19 18 17 16 16 03/30 Read block 31 len 8192 04/30 Read block 59 len 28672 05/30 Write block 579 len 32768 data: 7 7 6 6 10 11 12 12 06/30 Read block 453 len 4096 07/30 Read block 389 len 12288 08/30 Write block 98 len 4096 data: 7 09/30 Write block 5 len 32768 data: 7 8 7 6 6 5 5 5 10/30 Write block 344 len 4096 data: 15 11/30 Write block 66 len 12288 data: 10 8 7 12/30 Read block 172 len 32768 13/30 Read block 348 len 36864 14/30 Read block 350 len 28672 15/30 Read block 557 len 4096 16/30 Read block 391 len 28672 17/30 Write block 512 len 28672 data: 14 12 14 11 11 12 12 18/30 Read block 46 len 24576 19/30 Write block 270 len 36864 data: 11 13 14 13 13 12 12 12 12 20/30 Read block 499 len 28672 21/30 Write block 570 len 20480 data: 16 16 12 11 9 22/30 Write block 475 len 4096 data: 13 23/30 Write block 568 len 40960 data: 15 16 17 17 13 12 10 7 6 9 24/30 Write block 0 len 36864 data: 4 6 6 7 8 8 9 8 7 25/30 Write block 134 len 12288 data: 15 13 13 26/30 Read block 448 len 28672 27/30 Write block 456 len 4096 data: 8 28/30 Read block 590 len 40960 29/30 Read block 174 len 40960 30/30 Write block 195 len 40960 data: 9 10 10 6 6 6 7 10 14 15 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Write 1 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 8 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 7 Done Done Done false 1014 Acked Read 1 Done Done Done false 1015 Acked Read 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 6 Done Done Done false 1018 Acked Write 9 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 5 Done Done Done false 1021 Acked Write 1 Done Done Done false 1022 Acked Write 10 Done Done Done false 1023 Acked Write 9 Done Done Done false 1024 Acked Write 3 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Write 1 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Read 10 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 28 28 28 84 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1000 1000 1000 Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:36.640992865Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1774} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 132 132 132 227 227 227 T T T 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 145 145 145 245 245 245 T T T 4 080-099 147 147 147 250 250 250 T T T 5 100-119 146 146 146 249 249 249 F F F 6 120-139 143 143 143 241 241 241 T T T 7 140-159 146 146 146 246 246 246 F F F 8 160-179 147 147 147 250 250 250 F F F 9 180-199 144 144 144 243 243 243 T T T 10 200-219 146 146 146 246 246 246 T T T 11 220-239 136 136 136 228 228 228 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 147 147 147 250 250 250 T T T 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 145 145 145 245 245 245 T T T 17 340-359 145 145 145 245 245 245 T T T 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 146 146 146 248 248 248 T T T 23 460-479 146 146 146 248 248 248 T T T 24 480-499 145 145 145 245 245 245 F F F 25 500-519 145 145 145 245 245 245 T T T 26 520-539 146 146 146 246 246 246 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 142 142 142 240 240 240 T T T 29 580-599 140 140 140 237 237 237 T T T Max gen: 147, Max flush: 250 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 148 148 148 251 251 251 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 146 146 146 249 249 249 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 146 146 146 246 246 246 F F F 8 160-179 147 147 147 250 250 250 F F F 9 180-199 148 148 148 251 251 251 F F F 10 200-219 148 148 148 251 251 251 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 148 148 148 251 251 251 F F F 17 340-359 148 148 148 251 251 251 F F F 18 360-379 142 142 142 240 240 240 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 148 148 148 251 251 251 F F F 23 460-479 148 148 148 251 251 251 F F F 24 480-499 145 145 145 245 245 245 F F F 25 500-519 148 148 148 251 251 251 F F F 26 520-539 146 146 146 246 246 246 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 148 148 148 251 251 251 F F F 29 580-599 148 148 148 251 251 251 F F F Max gen: 148, Max flush: 251 Begin loop 75 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.110599406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.154759852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.155378449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.155419178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.155435218Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.155449657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} {"msg":"Crucible 32b149d9-6ba8-43c8-85f4-6f38315ce334 has session id: 470f6f7c-3375-4367-8027-90e85f616dd2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.15591247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Upstairs opts: Upstairs UUID: 32b149d9-6ba8-43c8-85f4-6f38315ce334, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.155954689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Crucible stats registered with UUID: 32b149d9-6ba8-43c8-85f4-6f38315ce334","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.155974928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186363361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186413919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} {"msg":"32b149d9-6ba8-43c8-85f4-6f38315ce334 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186485837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186521205Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186560484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186596473Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186630921Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.18667708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186718878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186754997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186788246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.186980709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:49749)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187024487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"io task","client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:62437)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187064716Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"io task","client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60324)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187104864Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"io task","client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187711052Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187738711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187753481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.1877674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.18778156Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.187795059Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.188345859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.188490974Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.18860955Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.188909339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.188945568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.188961107Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189044364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189065953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189079863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189232378Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189251567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[0]R flush_numbers[0..12]: [251, 245, 245, 251, 251, 249, 251, 246, 250, 251, 251, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189269726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[0]R generation[0..12]: [148, 145, 145, 148, 148, 146, 148, 146, 147, 148, 148, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189284076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189298395Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[1]R flush_numbers[0..12]: [251, 245, 245, 251, 251, 249, 251, 246, 250, 251, 251, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189312544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[1]R generation[0..12]: [148, 145, 145, 148, 148, 146, 148, 146, 147, 148, 148, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189326364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189340434Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[2]R flush_numbers[0..12]: [251, 245, 245, 251, 251, 249, 251, 246, 250, 251, 251, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189358253Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[2]R generation[0..12]: [148, 145, 145, 148, 148, 146, 148, 146, 147, 148, 148, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189372562Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189397452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Max found gen is 149","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.18942364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Generation requested: 150 >= found:149","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.18944952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Next flush: 252","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2024-11-07T14:08:41.189519847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189553736Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"":"downstairs","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189569785Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189582215Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"0","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189605784Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"1","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189632783Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"client":"2","":"downstairs","session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189658652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189682261Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"32b149d9-6ba8-43c8-85f4-6f38315ce334 is now active with session: 470f6f7c-3375-4367-8027-90e85f616dd2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.18970914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189727509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782,"session_id":"470f6f7c-3375-4367-8027-90e85f616dd2"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:41.189743649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 480 len 36864 data: 7 10 10 12 13 12 12 12 11 02/30 Flush 03/30 Write block 449 len 24576 data: 12 13 10 11 16 14 04/30 Read block 490 len 12288 05/30 Write block 213 len 4096 data: 10 06/30 Read block 529 len 32768 07/30 Read block 587 len 12288 08/30 Write block 107 len 40960 data: 13 14 14 15 15 16 17 15 15 15 09/30 Read block 422 len 4096 10/30 Read block 196 len 16384 11/30 Read block 342 len 20480 12/30 Write block 526 len 32768 data: 11 9 8 9 10 14 13 11 13/30 Read block 286 len 16384 14/30 Write block 73 len 28672 data: 6 9 10 12 11 9 9 15/30 Read block 341 len 32768 16/30 Read block 309 len 4096 17/30 Flush 18/30 Flush 19/30 Write block 461 len 36864 data: 16 16 16 14 15 14 13 11 11 20/30 Read block 592 len 4096 21/30 Flush 22/30 Write block 177 len 20480 data: 8 8 10 11 12 23/30 Write block 340 len 8192 data: 11 14 24/30 Read block 181 len 24576 25/30 Read block 578 len 16384 26/30 Read block 403 len 12288 27/30 Read block 506 len 24576 28/30 Write block 192 len 16384 data: 9 11 13 10 29/30 Write block 361 len 28672 data: 14 14 15 14 11 11 12 30/30 Write block 590 len 8192 data: 16 12 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 6 Done Done Done false 1024 Acked Read 4 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 4 Done Done Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 7 7 7 21 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:41.215979452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1782} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 150 150 150 253 253 253 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 150 150 150 253 253 253 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 146 146 146 246 246 246 F F F 8 160-179 147 147 147 250 250 250 T T T 9 180-199 148 148 148 251 251 251 T T T 10 200-219 150 150 150 253 253 253 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 148 148 148 251 251 251 F F F 17 340-359 148 148 148 251 251 251 T T T 18 360-379 142 142 142 240 240 240 T T T 19 380-399 141 141 141 239 239 239 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 150 150 150 253 253 253 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 150 150 150 252 252 252 F F F 25 500-519 148 148 148 251 251 251 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 148 148 148 251 251 251 F F F 29 580-599 148 148 148 251 251 251 T T T Max gen: 150, Max flush: 255 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 145 145 145 245 245 245 F F F 2 040-059 145 145 145 245 245 245 F F F 3 060-079 150 150 150 253 253 253 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 150 150 150 253 253 253 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 146 146 146 246 246 246 F F F 8 160-179 151 151 151 256 256 256 F F F 9 180-199 151 151 151 256 256 256 F F F 10 200-219 150 150 150 253 253 253 F F F 11 220-239 136 136 136 228 228 228 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 130 130 130 225 225 225 F F F 16 320-339 148 148 148 251 251 251 F F F 17 340-359 151 151 151 256 256 256 F F F 18 360-379 151 151 151 256 256 256 F F F 19 380-399 141 141 141 239 239 239 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 150 150 150 253 253 253 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 150 150 150 252 252 252 F F F 25 500-519 148 148 148 251 251 251 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 148 148 148 251 251 251 F F F 29 580-599 151 151 151 256 256 256 F F F Max gen: 151, Max flush: 256 Begin loop 76 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.541448447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:45.583244568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.583620605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.614399324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.614924755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.614960303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.614979923Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.614995082Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} {"msg":"Crucible 08c78747-b012-4e5c-b01b-cd9fb94d5486 has session id: e3040dc0-91cd-49ae-83b1-a638ea925926","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.615434146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Upstairs opts: Upstairs UUID: 08c78747-b012-4e5c-b01b-cd9fb94d5486, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.615464705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Crucible stats registered with UUID: 08c78747-b012-4e5c-b01b-cd9fb94d5486","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.615479205Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646161647Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646197466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} {"msg":"08c78747-b012-4e5c-b01b-cd9fb94d5486 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646216335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646233225Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646259684Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646277543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646348241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646395729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646431987Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646465536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646694918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646726547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_connection connected from Ok(127.0.0.1:61872)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646752046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"io task","client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_connection connected from Ok(127.0.0.1:54412)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646786665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"io task","client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_connection connected from Ok(127.0.0.1:36031)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.646819924Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"io task","client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.64718576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.647211129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.647373144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.647405732Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.647435711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.64746218Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.647896425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.647987461Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648259541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648455114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648485893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648501033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648517622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648531272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648544661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648739564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648762113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[0]R flush_numbers[0..12]: [251, 245, 245, 253, 251, 253, 251, 246, 256, 256, 253, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648776723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[0]R generation[0..12]: [148, 145, 145, 150, 148, 150, 148, 146, 151, 151, 150, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648790872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648805132Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[1]R flush_numbers[0..12]: [251, 245, 245, 253, 251, 253, 251, 246, 256, 256, 253, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648819261Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[1]R generation[0..12]: [148, 145, 145, 150, 148, 150, 148, 146, 151, 151, 150, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648833061Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.64884722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[2]R flush_numbers[0..12]: [251, 245, 245, 253, 251, 253, 251, 246, 256, 256, 253, 228]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.64886117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[2]R generation[0..12]: [148, 145, 145, 150, 148, 150, 148, 146, 151, 151, 150, 136]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648877889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648892369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648905658Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648919048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648932367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648945687Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"":"downstairs","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648968366Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648980555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"0","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.648994075Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"1","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.649007725Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"client":"2","":"downstairs","session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.649022144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.649034493Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"08c78747-b012-4e5c-b01b-cd9fb94d5486 is now active with session: e3040dc0-91cd-49ae-83b1-a638ea925926","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.649047463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.649059543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790,"session_id":"e3040dc0-91cd-49ae-83b1-a638ea925926"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:45.649071822Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 155 len 24576 data: 10 11 10 8 8 9 02/30 Write block 503 len 8192 data: 14 14 03/30 Read block 443 len 36864 04/30 Write block 587 len 36864 data: 13 15 17 17 13 12 12 12 9 05/30 Read block 249 len 40960 06/30 Write block 565 len 12288 data: 12 12 16 07/30 Read block 457 len 8192 08/30 Read block 109 len 40960 09/30 Read block 364 len 40960 10/30 Read block 438 len 4096 11/30 Write block 222 len 36864 data: 9 8 7 7 8 9 8 9 9 12/30 Read block 232 len 36864 13/30 Write block 31 len 12288 data: 19 15 14 14/30 Flush 15/30 Write block 307 len 4096 data: 9 16/30 Write block 499 len 32768 data: 16 16 13 14 15 15 11 11 17/30 Read block 420 len 16384 18/30 Write block 330 len 12288 data: 16 20 19 19/30 Read block 437 len 40960 20/30 Write block 397 len 4096 data: 18 21/30 Flush 22/30 Write block 218 len 40960 data: 13 12 9 9 10 9 8 8 9 10 23/30 Read block 165 len 24576 24/30 Read block 17 len 20480 25/30 Read block 376 len 12288 26/30 Read block 102 len 4096 27/30 Read block 66 len 4096 28/30 Write block 508 len 20480 data: 11 11 15 16 15 29/30 Read block 329 len 4096 30/30 Write block 56 len 20480 data: 11 11 13 12 14 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 10 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 5 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:45.674324291Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1790} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 152 152 152 257 257 257 F F F 2 040-059 145 145 145 245 245 245 T T T 3 060-079 150 150 150 253 253 253 T T T 4 080-099 148 148 148 251 251 251 F F F 5 100-119 150 150 150 253 253 253 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 152 152 152 257 257 257 F F F 8 160-179 152 152 152 257 257 257 F F F 9 180-199 151 151 151 256 256 256 F F F 10 200-219 150 150 150 253 253 253 T T T 11 220-239 152 152 152 257 257 257 T T T 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 152 152 152 258 258 258 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 151 151 151 256 256 256 F F F 18 360-379 151 151 151 256 256 256 F F F 19 380-399 152 152 152 258 258 258 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 150 150 150 253 253 253 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 152 152 152 258 258 258 T T T 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 152 152 152 257 257 257 F F F 29 580-599 152 152 152 257 257 257 F F F Max gen: 152, Max flush: 258 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 152 152 152 257 257 257 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 150 150 150 253 253 253 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 152 152 152 257 257 257 F F F 8 160-179 152 152 152 257 257 257 F F F 9 180-199 151 151 151 256 256 256 F F F 10 200-219 153 153 153 259 259 259 F F F 11 220-239 153 153 153 259 259 259 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 152 152 152 258 258 258 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 151 151 151 256 256 256 F F F 18 360-379 151 151 151 256 256 256 F F F 19 380-399 152 152 152 258 258 258 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 150 150 150 253 253 253 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 153 153 153 259 259 259 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 152 152 152 257 257 257 F F F 29 580-599 152 152 152 257 257 257 F F F Max gen: 153, Max flush: 259 Begin loop 77 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.027710938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:50.070664377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.071108252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.102496388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103044249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103083787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103100577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103121846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} {"msg":"Crucible 39509691-b39e-4f8b-86ad-87faeeba856a has session id: 79859731-cedf-4b95-a2ba-ff75d024b3fb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103604868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Upstairs opts: Upstairs UUID: 39509691-b39e-4f8b-86ad-87faeeba856a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103638507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Crucible stats registered with UUID: 39509691-b39e-4f8b-86ad-87faeeba856a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.103656246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.133949853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.133983402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} {"msg":"39509691-b39e-4f8b-86ad-87faeeba856a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.13403513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134055809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134083548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134101798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134119287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134135736Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134152816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134169475Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134186615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134221063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:37587)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134260072Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"io task","client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:33240)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.134341959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"io task","client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_connection connected from Ok(127.0.0.1:42045)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.13458281Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"io task","client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135010875Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135040504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135188978Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135223447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135251106Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135320643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135674881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.13570286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.135952221Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136266879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136298048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136317358Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136338687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136353246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136367056Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136433313Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136452773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[0]R flush_numbers[0..12]: [251, 257, 259, 259, 251, 253, 251, 257, 257, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136467232Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[0]R generation[0..12]: [148, 152, 153, 153, 148, 150, 148, 152, 152, 151, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136483272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136497941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[1]R flush_numbers[0..12]: [251, 257, 259, 259, 251, 253, 251, 257, 257, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136512771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[1]R generation[0..12]: [148, 152, 153, 153, 148, 150, 148, 152, 152, 151, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.1365266Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136540719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[2]R flush_numbers[0..12]: [251, 257, 259, 259, 251, 253, 251, 257, 257, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136556029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[2]R generation[0..12]: [148, 152, 153, 153, 148, 150, 148, 152, 152, 151, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136571508Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136588328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136602057Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136615287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Next flush: 260","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136628286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136641236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"":"downstairs","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:08:50.136654486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136735743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"0","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136753042Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"1","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136766621Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"client":"2","":"downstairs","session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136779981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136792581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"39509691-b39e-4f8b-86ad-87faeeba856a is now active with session: 79859731-cedf-4b95-a2ba-ff75d024b3fb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.13680561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798,"session_id":"79859731-cedf-4b95-a2ba-ff75d024b3fb"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:50.136830469Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 362 len 28672 data: 15 16 15 12 12 13 11 02/30 Read block 358 len 40960 03/30 Read block 349 len 40960 04/30 Read block 582 len 8192 05/30 Read block 304 len 8192 06/30 Write block 392 len 4096 data: 13 07/30 Read block 223 len 32768 08/30 Read block 99 len 12288 09/30 Write block 28 len 8192 data: 15 17 10/30 Write block 341 len 36864 data: 15 16 16 16 15 13 13 16 16 11/30 Write block 103 len 8192 data: 12 13 12/30 Read block 124 len 36864 13/30 Write block 386 len 16384 data: 12 9 11 10 14/30 Flush 15/30 Flush 16/30 Write block 593 len 12288 data: 13 13 10 17/30 Read block 566 len 32768 18/30 Read block 366 len 16384 19/30 Read block 210 len 28672 20/30 Read block 123 len 28672 21/30 Write block 25 len 8192 data: 16 15 22/30 Write block 164 len 40960 data: 11 13 14 12 12 9 9 9 8 7 23/30 Read block 260 len 8192 24/30 Read block 546 len 12288 25/30 Read block 120 len 12288 26/30 Read block 241 len 12288 27/30 Write block 454 len 16384 data: 15 12 9 11 28/30 Write block 352 len 36864 data: 10 8 11 12 12 13 12 12 15 29/30 Read block 535 len 20480 30/30 Read block 119 len 12288 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:15 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1015 Acked Write 3 Done Done Done false 1016 Acked Read 8 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 7 Done Done Done false 1019 Acked Read 7 Done Done Done false 1020 Acked Write 2 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Read 2 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Write 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 3 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 15 15 15 45 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1014 1014 1014 Downstairs last five completed: 1014 1013 1012 1011 1010 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:50.159928165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1798} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 154 154 154 260 260 260 T T T 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 154 154 154 260 260 260 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 152 152 152 257 257 257 F F F 8 160-179 152 152 152 257 257 257 T T T 9 180-199 151 151 151 256 256 256 F F F 10 200-219 153 153 153 259 259 259 F F F 11 220-239 153 153 153 259 259 259 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 152 152 152 258 258 258 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 154 154 154 260 260 260 T T T 18 360-379 154 154 154 260 260 260 T T T 19 380-399 154 154 154 260 260 260 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 150 150 150 253 253 253 T T T 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 153 153 153 259 259 259 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 152 152 152 257 257 257 F F F 29 580-599 152 152 152 257 257 257 T T T Max gen: 154, Max flush: 260 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 154 154 154 260 260 260 F F F 6 120-139 148 148 148 251 251 251 F F F 7 140-159 152 152 152 257 257 257 F F F 8 160-179 155 155 155 261 261 261 F F F 9 180-199 151 151 151 256 256 256 F F F 10 200-219 153 153 153 259 259 259 F F F 11 220-239 153 153 153 259 259 259 F F F 12 240-259 147 147 147 250 250 250 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 F F F 15 300-319 152 152 152 258 258 258 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 155 155 155 261 261 261 F F F 18 360-379 155 155 155 261 261 261 F F F 19 380-399 154 154 154 260 260 260 F F F 20 400-419 146 146 146 247 247 247 F F F 21 420-439 137 137 137 231 231 231 F F F 22 440-459 155 155 155 261 261 261 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 153 153 153 259 259 259 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 152 152 152 257 257 257 F F F 29 580-599 155 155 155 261 261 261 F F F Max gen: 155, Max flush: 261 Begin loop 78 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.51753122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:54.559621521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.560026596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.590920431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.591557678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.591592087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.591606006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.591625326Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} {"msg":"Crucible 268fd66d-d46d-4479-8ed7-6d1f1a8901d0 has session id: 25b313f2-3de7-4580-b10d-094f6c4805ed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.59206194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Upstairs opts: Upstairs UUID: 268fd66d-d46d-4479-8ed7-6d1f1a8901d0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.592102088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Crucible stats registered with UUID: 268fd66d-d46d-4479-8ed7-6d1f1a8901d0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.592128688Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621671871Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621715699Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} {"msg":"268fd66d-d46d-4479-8ed7-6d1f1a8901d0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621746328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621776727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621824296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621862094Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621897813Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.621931092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.6219678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.622002409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.622036648Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.622075866Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:43874)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.622112275Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"io task","client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:53195)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.622150944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"io task","client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:50536)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.622188612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"io task","client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623036342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623065281Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.6230807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.62309471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623108509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623121829Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623711127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623753856Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.623786155Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624294776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624324435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624343874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624357624Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624372193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624386013Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624398452Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624411832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[0]R flush_numbers[0..12]: [251, 261, 259, 259, 251, 260, 251, 257, 261, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624425722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[0]R generation[0..12]: [148, 155, 153, 153, 148, 154, 148, 152, 155, 151, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624439671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.62445389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[1]R flush_numbers[0..12]: [251, 261, 259, 259, 251, 260, 251, 257, 261, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.62446802Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[1]R generation[0..12]: [148, 155, 153, 153, 148, 154, 148, 152, 155, 151, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624481929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624495999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[2]R flush_numbers[0..12]: [251, 261, 259, 259, 251, 260, 251, 257, 261, 256, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624509768Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[2]R generation[0..12]: [148, 155, 153, 153, 148, 154, 148, 152, 155, 151, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624523458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624609355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624626144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624639644Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Next flush: 262","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624652743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624666023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"":"downstairs","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624679522Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624691792Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"0","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624705012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"1","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624718241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"client":"2","":"downstairs","session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.62473162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.62474406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"268fd66d-d46d-4479-8ed7-6d1f1a8901d0 is now active with session: 25b313f2-3de7-4580-b10d-094f6c4805ed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624757099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624769339Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806,"session_id":"25b313f2-3de7-4580-b10d-094f6c4805ed"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:54.624782429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 427 len 36864 data: 10 9 11 10 9 11 11 11 10 02/30 Write block 100 len 32768 data: 6 9 12 13 14 15 16 14 03/30 Read block 51 len 20480 04/30 Read block 493 len 36864 05/30 Read block 362 len 20480 06/30 Read block 544 len 24576 07/30 Read block 263 len 8192 08/30 Read block 357 len 12288 09/30 Write block 587 len 20480 data: 14 16 18 18 14 10/30 Read block 257 len 28672 11/30 Flush 12/30 Write block 582 len 20480 data: 7 11 12 13 13 13/30 Write block 129 len 36864 data: 15 15 15 17 12 16 14 14 13 14/30 Read block 246 len 36864 15/30 Read block 413 len 16384 16/30 Write block 416 len 4096 data: 9 17/30 Write block 4 len 8192 data: 9 9 18/30 Read block 298 len 32768 19/30 Read block 507 len 8192 20/30 Write block 241 len 20480 data: 7 9 10 14 13 21/30 Write block 159 len 24576 data: 9 10 8 11 11 12 22/30 Write block 149 len 24576 data: 12 11 10 11 8 8 23/30 Write block 189 len 12288 data: 13 10 9 24/30 Read block 130 len 8192 25/30 Write block 296 len 36864 data: 13 13 14 11 7 8 8 10 8 26/30 Read block 107 len 20480 27/30 Read block 355 len 8192 28/30 Read block 80 len 4096 29/30 Read block 205 len 8192 30/30 Write block 114 len 8192 data: 16 16 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 5 Done Done Done false 1012 Acked Write 9 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 4 Done Done Done false 1015 Acked Write 1 Done Done Done false 1016 Acked Write 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Read 2 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Write 6 Done Done Done false 1022 Acked Write 3 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 1 Done Done Done false 1028 Acked Read 2 Done Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 18 18 18 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1010 1010 1010 Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:54.64552099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1806} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 251 251 251 T T T 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 156 156 156 262 262 262 T T T 6 120-139 148 148 148 251 251 251 T T T 7 140-159 152 152 152 257 257 257 T T T 8 160-179 155 155 155 261 261 261 T T T 9 180-199 151 151 151 256 256 256 T T T 10 200-219 153 153 153 259 259 259 F F F 11 220-239 153 153 153 259 259 259 F F F 12 240-259 147 147 147 250 250 250 T T T 13 260-279 148 148 148 251 251 251 F F F 14 280-299 144 144 144 244 244 244 T T T 15 300-319 152 152 152 258 258 258 T T T 16 320-339 152 152 152 258 258 258 F F F 17 340-359 155 155 155 261 261 261 F F F 18 360-379 155 155 155 261 261 261 F F F 19 380-399 154 154 154 260 260 260 F F F 20 400-419 146 146 146 247 247 247 T T T 21 420-439 156 156 156 262 262 262 F F F 22 440-459 155 155 155 261 261 261 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 153 153 153 259 259 259 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 152 152 152 257 257 257 F F F 29 580-599 156 156 156 262 262 262 T T T Max gen: 156, Max flush: 262 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 157 157 157 263 263 263 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 148 148 148 251 251 251 F F F 5 100-119 157 157 157 263 263 263 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 157 157 157 263 263 263 F F F 8 160-179 157 157 157 263 263 263 F F F 9 180-199 157 157 157 263 263 263 F F F 10 200-219 153 153 153 259 259 259 F F F 11 220-239 153 153 153 259 259 259 F F F 12 240-259 157 157 157 263 263 263 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 157 157 157 263 263 263 F F F 15 300-319 157 157 157 263 263 263 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 155 155 155 261 261 261 F F F 18 360-379 155 155 155 261 261 261 F F F 19 380-399 154 154 154 260 260 260 F F F 20 400-419 157 157 157 263 263 263 F F F 21 420-439 156 156 156 262 262 262 F F F 22 440-459 155 155 155 261 261 261 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 153 153 153 259 259 259 F F F 26 520-539 150 150 150 253 253 253 F F F 27 540-559 143 143 143 241 241 241 F F F 28 560-579 152 152 152 257 257 257 F F F 29 580-599 157 157 157 263 263 263 F F F Max gen: 157, Max flush: 263 Begin loop 79 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.064008538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.107303025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.107910613Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.107956631Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.10797512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.10799324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} {"msg":"Crucible 37851394-0767-4522-a628-2f525c68f434 has session id: 31d95116-ebc2-476e-87e2-aaa2b545d53a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.108503491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Upstairs opts: Upstairs UUID: 37851394-0767-4522-a628-2f525c68f434, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.10854572Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Crucible stats registered with UUID: 37851394-0767-4522-a628-2f525c68f434","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.108571809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138263717Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138301166Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} {"msg":"37851394-0767-4522-a628-2f525c68f434 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138339274Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138360483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138404872Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138440361Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.13847573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138512568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138550777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138582855Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138612445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138645853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_connection connected from Ok(127.0.0.1:59332)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138679362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"io task","client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_connection connected from Ok(127.0.0.1:43591)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138798168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"io task","client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_connection connected from Ok(127.0.0.1:55076)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.138818517Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"io task","client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139359467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139389426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139405196Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139431025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139459204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139489243Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.139963146Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140006364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.1401176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140568994Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140606913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140634321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140656161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.14067067Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.14068435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140696759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140710329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[0]R flush_numbers[0..12]: [263, 261, 259, 259, 251, 263, 263, 263, 263, 263, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140724508Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[0]R generation[0..12]: [157, 155, 153, 153, 148, 157, 157, 157, 157, 157, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140738918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140753077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[1]R flush_numbers[0..12]: [263, 261, 259, 259, 251, 263, 263, 263, 263, 263, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140766997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[1]R generation[0..12]: [157, 155, 153, 153, 148, 157, 157, 157, 157, 157, 153, 153]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140780836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140794826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[2]R flush_numbers[0..12]: [263, 261, 259, 259, 251, 263, 263, 263, 263, 263, 259, 259]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140808655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[2]R generation[0..12]: [157, 155, 153, 153, 148, 157, 157, 157, 157, 157, 153, 153]",Wait for a query_work_queue command to finish before sending IO "v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140832044Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140859603Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140889372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140916981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Next flush: 264","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.14094289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.14096035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"":"downstairs","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140974079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140986249Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"0","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.140999548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"1","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.141012878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"client":"2","":"downstairs","session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:08:59.141026147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.141038927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"37851394-0767-4522-a628-2f525c68f434 is now active with session: 31d95116-ebc2-476e-87e2-aaa2b545d53a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.141051746Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.141063656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814,"session_id":"31d95116-ebc2-476e-87e2-aaa2b545d53a"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:08:59.141075965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 432 len 12288 data: 12 12 12 03/30 Write block 450 len 36864 data: 14 11 12 17 16 13 10 12 11 04/30 Write block 95 len 8192 data: 14 13 05/30 Read block 481 len 40960 06/30 Write block 1 len 8192 data: 7 7 07/30 Read block 402 len 16384 08/30 Write block 534 len 20480 data: 11 10 9 10 11 09/30 Write block 175 len 16384 data: 9 8 9 9 10/30 Write block 233 len 4096 data: 7 11/30 Write block 503 len 24576 data: 16 16 12 12 10 12 12/30 Read block 216 len 16384 13/30 Read block 549 len 36864 14/30 Read block 15 len 16384 15/30 Write block 547 len 32768 data: 16 20 18 17 17 18 20 14 16/30 Write block 544 len 24576 data: 10 10 15 17 21 19 17/30 Flush 18/30 Write block 284 len 28672 data: 17 17 18 18 17 13 10 19/30 Read block 500 len 24576 20/30 Write block 5 len 20480 data: 10 10 9 8 7 21/30 Read block 531 len 40960 22/30 Write block 579 len 40960 data: 8 8 7 8 12 13 14 14 15 17 23/30 Write block 436 len 16384 data: 9 9 8 6 24/30 Read block 176 len 20480 25/30 Read block 210 len 36864 26/30 Read block 292 len 16384 27/30 Read block 450 len 12288 28/30 Write block 250 len 36864 data: 14 17 18 16 16 18 18 15 12 29/30 Read block 212 len 12288 30/30 Write block 6 len 32768 data: 11 10 9 8 6 6 6 9 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 7 Done Done Done false 1018 Acked Read 6 Done Done Done false 1019 Acked Write 5 Done Done Done false 1020 Acked Read 10 Done Done Done false 1021 Acked Write 10 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:08:59.163266284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1814} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 265 265 265 T T T 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 158 158 158 265 265 265 F F F 5 100-119 157 157 157 263 263 263 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 157 157 157 263 263 263 F F F 8 160-179 158 158 158 265 265 265 F F F 9 180-199 157 157 157 263 263 263 F F F 10 200-219 153 153 153 259 259 259 F F F 11 220-239 158 158 158 265 265 265 F F F 12 240-259 157 157 157 263 263 263 T T T 13 260-279 148 148 148 251 251 251 F F F 14 280-299 157 157 157 263 263 263 T T T 15 300-319 157 157 157 263 263 263 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 155 155 155 261 261 261 F F F 18 360-379 155 155 155 261 261 261 F F F 19 380-399 154 154 154 260 260 260 F F F 20 400-419 157 157 157 263 263 263 F F F 21 420-439 158 158 158 265 265 265 T T T 22 440-459 158 158 158 265 265 265 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 158 158 158 265 265 265 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 152 152 152 257 257 257 T T T 29 580-599 157 157 157 263 263 263 T T T Max gen: 158, Max flush: 265 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 266 266 266 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 158 158 158 265 265 265 F F F 5 100-119 157 157 157 263 263 263 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 157 157 157 263 263 263 F F F 8 160-179 158 158 158 265 265 265 F F F 9 180-199 157 157 157 263 263 263 F F F 10 200-219 153 153 153 259 259 259 F F F 11 220-239 158 158 158 265 265 265 F F F 12 240-259 159 159 159 266 266 266 F F F 13 260-279 148 148 148 251 251 251 F F F 14 280-299 159 159 159 266 266 266 F F F 15 300-319 157 157 157 263 263 263 F F F 16 320-339 152 152 152 258 258 258 F F F 17 340-359 155 155 155 261 261 261 F F F 18 360-379 155 155 155 261 261 261 F F F 19 380-399 154 154 154 260 260 260 F F F 20 400-419 157 157 157 263 263 263 F F F 21 420-439 159 159 159 266 266 266 F F F 22 440-459 158 158 158 265 265 265 F F F 23 460-479 150 150 150 255 255 255 F F F 24 480-499 152 152 152 258 258 258 F F F 25 500-519 158 158 158 265 265 265 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 159, Max flush: 266 Begin loop 80 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.492074969Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:03.534259506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.534658422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.565492589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566040959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566076288Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566090307Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566104757Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} {"msg":"Crucible 4f493790-34f1-413c-a0e1-52f9caecb545 has session id: 35670556-cd00-40fb-82c8-bc656556e62d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566634498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Upstairs opts: Upstairs UUID: 4f493790-34f1-413c-a0e1-52f9caecb545, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566678276Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Crucible stats registered with UUID: 4f493790-34f1-413c-a0e1-52f9caecb545","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.566700675Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596242369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596299117Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} {"msg":"4f493790-34f1-413c-a0e1-52f9caecb545 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596321426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596338515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596367105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596399173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596433612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596463661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.59649466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596523539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596550278Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596576847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_connection connected from Ok(127.0.0.1:56504)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596682923Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"io task","client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_connection connected from Ok(127.0.0.1:33388)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596876096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"io task","client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_connection connected from Ok(127.0.0.1:43398)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.596918345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"io task","client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.597571081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.597611629Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.597642998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.597673547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.597704836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.597736125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598225057Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598261186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598291655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598870634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598910853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598939481Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.598965521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.59899077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599015779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599051068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599076687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[0]R flush_numbers[0..12]: [266, 261, 259, 259, 265, 263, 263, 263, 265, 263, 259, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599100876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[0]R generation[0..12]: [159, 155, 153, 153, 158, 157, 157, 157, 158, 157, 153, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599126965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599155264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[1]R flush_numbers[0..12]: [266, 261, 259, 259, 265, 263, 263, 263, 265, 263, 259, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599182383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[1]R generation[0..12]: [159, 155, 153, 153, 158, 157, 157, 157, 158, 157, 153, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599208012Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599236811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[2]R flush_numbers[0..12]: [266, 261, 259, 259, 265, 263, 263, 263, 265, 263, 259, 265]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:09:03.59926742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[2]R generation[0..12]: [159, 155, 153, 153, 158, 157, 157, 157, 158, 157, 153, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599297279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599328538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599360946Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599390745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Next flush: 267","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599418804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599449403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"":"downstairs","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599479622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599507551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"0","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.59952604Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"1","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.59953995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"client":"2","":"downstairs","session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.59955337Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599565999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"4f493790-34f1-413c-a0e1-52f9caecb545 is now active with session: 35670556-cd00-40fb-82c8-bc656556e62d","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599579098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599591448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822,"session_id":"35670556-cd00-40fb-82c8-bc656556e62d"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:03.599603837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 329 len 8192 data: 13 17 02/30 Flush 03/30 Write block 474 len 12288 data: 15 14 9 04/30 Read block 127 len 36864 05/30 Read block 338 len 28672 06/30 Flush 07/30 Write block 464 len 36864 data: 15 16 15 14 12 12 11 11 10 08/30 Flush 09/30 Write block 372 len 36864 data: 15 14 14 14 14 14 14 14 15 10/30 Read block 509 len 12288 11/30 Read block 206 len 16384 12/30 Write block 497 len 40960 data: 14 12 17 17 14 15 17 17 13 13 13/30 Write block 208 len 40960 data: 12 12 11 9 7 11 7 9 16 15 14/30 Write block 268 len 40960 data: 10 10 12 14 15 14 14 13 13 13 15/30 Write block 189 len 28672 data: 14 11 10 10 12 14 11 16/30 Read block 299 len 4096 17/30 Flush 18/30 Write block 246 len 16384 data: 13 11 13 14 19/30 Read block 415 len 36864 20/30 Write block 145 len 36864 data: 19 20 21 15 13 12 11 12 9 21/30 Write block 458 len 24576 data: 12 15 12 17 17 17 22/30 Read block 93 len 16384 23/30 Read block 423 len 24576 24/30 Write block 358 len 16384 data: 13 13 16 15 25/30 Read block 489 len 36864 26/30 Write block 84 len 20480 data: 7 11 14 18 19 27/30 Write block 438 len 12288 data: 9 7 7 28/30 Read block 434 len 8192 29/30 Write block 418 len 8192 data: 9 9 30/30 Write block 279 len 32768 data: 12 13 13 15 20 18 18 19 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 4 Done Done Done false 1018 Acked Read 9 Done Done Done false 1019 Acked Write 9 Done Done Done false 1020 Acked Write 6 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 4 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Write 5 Done Done Done false 1026 Acked Write 3 Done Done Done false 1027 Acked Read 2 Done Done Done false 1028 Acked Write 2 Done Done Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 12 12 12 36 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:03.625233582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1822} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 266 266 266 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 158 158 158 265 265 265 T T T 5 100-119 157 157 157 263 263 263 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 157 157 157 263 263 263 T T T 8 160-179 158 158 158 265 265 265 F F F 9 180-199 160 160 160 270 270 270 F F F 10 200-219 160 160 160 270 270 270 F F F 11 220-239 158 158 158 265 265 265 F F F 12 240-259 159 159 159 266 266 266 T T T 13 260-279 160 160 160 270 270 270 T T T 14 280-299 159 159 159 266 266 266 T T T 15 300-319 157 157 157 263 263 263 F F F 16 320-339 160 160 160 267 267 267 F F F 17 340-359 155 155 155 261 261 261 T T T 18 360-379 160 160 160 270 270 270 T T T 19 380-399 160 160 160 270 270 270 F F F 20 400-419 157 157 157 263 263 263 T T T 21 420-439 159 159 159 266 266 266 T T T 22 440-459 158 158 158 265 265 265 T T T 23 460-479 160 160 160 269 269 269 T T T 24 480-499 160 160 160 270 270 270 F F F 25 500-519 160 160 160 270 270 270 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 160, Max flush: 270 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 266 266 266 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 153 153 153 259 259 259 F F F 4 080-099 161 161 161 271 271 271 F F F 5 100-119 157 157 157 263 263 263 F F F 6 120-139 157 157 157 263 263 263 F F F 7 140-159 161 161 161 271 271 271 F F F 8 160-179 158 158 158 265 265 265 F F F 9 180-199 160 160 160 270 270 270 F F F 10 200-219 160 160 160 270 270 270 F F F 11 220-239 158 158 158 265 265 265 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 161 161 161 271 271 271 F F F 14 280-299 161 161 161 271 271 271 F F F 15 300-319 157 157 157 263 263 263 F F F 16 320-339 160 160 160 267 267 267 F F F 17 340-359 161 161 161 271 271 271 F F F 18 360-379 161 161 161 271 271 271 F F F 19 380-399 160 160 160 270 270 270 F F F 20 400-419 161 161 161 271 271 271 F F F 21 420-439 161 161 161 271 271 271 F F F 22 440-459 161 161 161 271 271 271 F F F 23 460-479 161 161 161 271 271 271 F F F 24 480-499 160 160 160 270 270 270 F F F 25 500-519 160 160 160 270 270 270 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 161, Max flush: 271 Begin loop 81 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.048022755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:08.090967715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.09138238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.122799096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123360075Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123403834Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123419633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123434203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} {"msg":"Crucible 6e10992b-b90d-4fc0-8900-a6a3b817fdf6 has session id: 098b40e3-3b87-4857-8de8-f47ca4ffbbcb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123818779Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Upstairs opts: Upstairs UUID: 6e10992b-b90d-4fc0-8900-a6a3b817fdf6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123852568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Crucible stats registered with UUID: 6e10992b-b90d-4fc0-8900-a6a3b817fdf6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.123868367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154297549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154329717Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} {"msg":"6e10992b-b90d-4fc0-8900-a6a3b817fdf6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154380126Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154400115Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154427254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154445143Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154469382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154502421Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.15453665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154571719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154606937Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154660165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:59915)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154703344Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"io task","client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:51737)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154753822Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"io task","client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_connection connected from Ok(127.0.0.1:43586)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.154842239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"io task","client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.155458167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.155487846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.155504605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.155519034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.155596242Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.155616541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156096914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156229719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156258258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156645384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156681632Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156712661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.15673555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.15675103Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.15676498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156781709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156795418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[0]R flush_numbers[0..12]: [266, 261, 259, 259, 271, 263, 263, 271, 265, 270, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156824147Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[0]R generation[0..12]: [159, 155, 153, 153, 161, 157, 157, 161, 158, 160, 160, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156838867Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156853196Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[1]R flush_numbers[0..12]: [266, 261, 259, 259, 271, 263, 263, 271, 265, 270, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156867406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[1]R generation[0..12]: [159, 155, 153, 153, 161, 157, 157, 161, 158, 160, 160, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156881195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156895425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[2]R flush_numbers[0..12]: [266, 261, 259, 259, 271, 263, 263, 271, 265, 270, 270, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156909474Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[2]R generation[0..12]: [159, 155, 153, 153, 161, 157, 157, 161, 158, 160, 160, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156923714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156937953Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156954863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156968952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Next flush: 272","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.156982252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:09:08.156995581Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"":"downstairs","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.1570178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.15703064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"0","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157044399Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"1","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157058199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"client":"2","":"downstairs","session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157071678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157084388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"6e10992b-b90d-4fc0-8900-a6a3b817fdf6 is now active with session: 098b40e3-3b87-4857-8de8-f47ca4ffbbcb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157097377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157109447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830,"session_id":"098b40e3-3b87-4857-8de8-f47ca4ffbbcb"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:08.157121977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 483 len 4096 data: 13 02/30 Write block 285 len 12288 data: 19 20 19 03/30 Write block 321 len 32768 data: 13 13 11 10 11 11 11 13 04/30 Read block 53 len 32768 05/30 Read block 425 len 8192 06/30 Read block 82 len 36864 07/30 Read block 183 len 4096 08/30 Write block 97 len 20480 data: 11 8 5 7 10 09/30 Write block 215 len 20480 data: 10 17 16 14 13 10/30 Write block 110 len 24576 data: 16 16 17 18 17 17 11/30 Read block 375 len 8192 12/30 Flush 13/30 Write block 269 len 20480 data: 11 13 15 16 15 14/30 Read block 187 len 28672 15/30 Write block 66 len 32768 data: 11 9 8 9 10 10 8 7 16/30 Write block 138 len 40960 data: 10 12 11 7 9 12 14 20 21 22 17/30 Read block 335 len 12288 18/30 Read block 514 len 24576 19/30 Read block 462 len 20480 20/30 Read block 356 len 32768 21/30 Read block 65 len 4096 22/30 Write block 448 len 24576 data: 10 13 15 12 13 18 23/30 Read block 288 len 28672 24/30 Read block 540 len 36864 25/30 Write block 165 len 12288 data: 14 15 13 26/30 Flush 27/30 Write block 421 len 40960 data: 11 10 14 9 12 10 11 10 12 11 28/30 Read block 317 len 16384 29/30 Write block 287 len 36864 data: 20 18 14 11 12 8 10 8 11 30/30 Write block 117 len 16384 data: 15 15 15 16 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:4 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1026 Acked Write 10 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 2 2 2 6 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1025 1025 1025 Downstairs last five completed: 1025 1024 1023 1022 1021 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:08.180251862Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1830} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 266 266 266 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 162 162 162 273 273 273 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 162 162 162 272 272 272 T T T 6 120-139 162 162 162 273 273 273 T T T 7 140-159 162 162 162 273 273 273 F F F 8 160-179 162 162 162 273 273 273 F F F 9 180-199 160 160 160 270 270 270 F F F 10 200-219 162 162 162 272 272 272 F F F 11 220-239 158 158 158 265 265 265 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 162 162 162 272 272 272 T T T 15 300-319 157 157 157 263 263 263 F F F 16 320-339 162 162 162 272 272 272 F F F 17 340-359 161 161 161 271 271 271 F F F 18 360-379 161 161 161 271 271 271 F F F 19 380-399 160 160 160 270 270 270 F F F 20 400-419 161 161 161 271 271 271 F F F 21 420-439 161 161 161 271 271 271 T T T 22 440-459 162 162 162 273 273 273 F F F 23 460-479 161 161 161 271 271 271 F F F 24 480-499 162 162 162 272 272 272 F F F 25 500-519 160 160 160 270 270 270 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 162, Max flush: 273 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 159 159 159 266 266 266 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 162 162 162 273 273 273 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 163 163 163 274 274 274 F F F 6 120-139 163 163 163 274 274 274 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 162 162 162 273 273 273 F F F 9 180-199 160 160 160 270 270 270 F F F 10 200-219 162 162 162 272 272 272 F F F 11 220-239 158 158 158 265 265 265 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 163 163 163 274 274 274 F F F 15 300-319 157 157 157 263 263 263 F F F 16 320-339 162 162 162 272 272 272 F F F 17 340-359 161 161 161 271 271 271 F F F 18 360-379 161 161 161 271 271 271 F F F 19 380-399 160 160 160 270 270 270 F F F 20 400-419 161 161 161 271 271 271 F F F 21 420-439 163 163 163 274 274 274 F F F 22 440-459 162 162 162 273 273 273 F F F 23 460-479 161 161 161 271 271 271 F F F 24 480-499 162 162 162 272 272 272 F F F 25 500-519 160 160 160 270 270 270 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 163, Max flush: 274 Begin loop 82 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.508937233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:12.55167295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.552080125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.583284549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.583817879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.583853538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.583867758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.583882387Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} {"msg":"Crucible d0cf2d94-fdda-454e-8c0d-19fc2053038e has session id: 79b5a806-27fd-4204-a42c-942659eeefa9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.584338151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Upstairs opts: Upstairs UUID: d0cf2d94-fdda-454e-8c0d-19fc2053038e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.584387899Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Crucible stats registered with UUID: d0cf2d94-fdda-454e-8c0d-19fc2053038e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.584407138Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614261181Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.61429369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} {"msg":"d0cf2d94-fdda-454e-8c0d-19fc2053038e active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614357907Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614386766Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614413565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614428615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614442804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614456354Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614469933Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614483863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614509402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614538101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61364)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614568409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"io task","client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:64192)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614604068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"io task","client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_connection connected from Ok(127.0.0.1:63519)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.614637077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"io task","client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.615340422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.6153831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.615419939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.615452628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.615482047Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.615511375Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.615977699Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616009568Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616127873Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616576317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616606786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616623015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616642935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616657134Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616675464Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616735571Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616754961Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[0]R flush_numbers[0..12]: [266, 261, 259, 273, 272, 274, 274, 273, 273, 270, 272, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.61676968Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[0]R generation[0..12]: [159, 155, 153, 162, 162, 163, 163, 162, 162, 160, 162, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616783989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616798279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[1]R flush_numbers[0..12]: [266, 261, 259, 273, 272, 274, 274, 273, 273, 270, 272, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616812099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[1]R generation[0..12]: [159, 155, 153, 162, 162, 163, 163, 162, 162, 160, 162, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616825758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616839787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[2]R flush_numbers[0..12]: [266, 261, 259, 273, 272, 274, 274, 273, 273, 270, 272, 265]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616853577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[2]R generation[0..12]: [159, 155, 153, 162, 162, 163, 163, 162, 162, 160, 162, 158]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616867357Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616881316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616894605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616908035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Next flush: 275","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616924624Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616938034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"":"downstairs","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616959403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616971543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"0","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.616984662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"1","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.617014711Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"client":"2","":"downstairs","session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.617030141Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.61704254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"d0cf2d94-fdda-454e-8c0d-19fc2053038e is now active with session: 79b5a806-27fd-4204-a42c-942659eeefa9","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.6170555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.617067729Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838,"session_id":"79b5a806-27fd-4204-a42c-942659eeefa9"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:12.617079989Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 12288 data: 7 10 10 02/30 Read block 258 len 20480 03/30 Read block 505 len 40960 04/30 Write block 359 len 20480 data: 14 17 16 16 17 05/30 Read block 244 len 8192 06/30 Read block 544 len 16384 07/30 Write block 310 len 8192 data: 9 8 08/30 Read block 189 len 32768 09/30 Read block 159 len 36864 10/30 Write block 227 len 4096 data: 11 11/30 Read block 72 len 36864 12/30 Write block 499 len 12288 data: 18 18 15 13/30 Write block 229 len 8192 data: 10 10 14/30 Read block 360 len 24576 15/30 Read block 151 len 12288 16/30 Write block 179 len 12288 data: 11 12 13 17/30 Write block 485 len 40960 data: 13 13 13 12 10 9 10 11 10 14 18/30 Read block 505 len 4096 19/30 Write block 71 len 28672 data: 11 9 8 10 11 13 12 20/30 Flush 21/30 Write block 394 len 16384 data: 15 18 18 19 22/30 Read block 248 len 36864 23/30 Write block 457 len 8192 data: 13 13 24/30 Read block 34 len 40960 25/30 Read block 400 len 40960 26/30 Read block 239 len 16384 27/30 Write block 167 len 20480 data: 14 13 10 10 10 28/30 Write block 291 len 40960 data: 13 9 11 9 12 14 14 15 12 8 29/30 Read block 229 len 20480 30/30 Write block 111 len 36864 data: 17 18 19 18 18 16 16 16 16 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 4 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Write 2 Done Done Done false 1023 Acked Read 10 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Write 5 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 9 9 9 27 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:12.638528055Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1838} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 275 275 275 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 164 164 164 275 275 275 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 163 163 163 274 274 274 T T T 6 120-139 163 163 163 274 274 274 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 164 164 164 275 275 275 T T T 9 180-199 164 164 164 275 275 275 F F F 10 200-219 162 162 162 272 272 272 F F F 11 220-239 164 164 164 275 275 275 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 163 163 163 274 274 274 T T T 15 300-319 164 164 164 275 275 275 T T T 16 320-339 162 162 162 272 272 272 F F F 17 340-359 164 164 164 275 275 275 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 160 160 160 270 270 270 T T T 20 400-419 161 161 161 271 271 271 F F F 21 420-439 163 163 163 274 274 274 F F F 22 440-459 162 162 162 273 273 273 T T T 23 460-479 161 161 161 271 271 271 F F F 24 480-499 164 164 164 275 275 275 F F F 25 500-519 164 164 164 275 275 275 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 164, Max flush: 275 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 164 164 164 275 275 275 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 153 153 153 259 259 259 F F F 3 060-079 164 164 164 275 275 275 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 165 165 165 276 276 276 F F F 6 120-139 163 163 163 274 274 274 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 165 165 165 276 276 276 F F F 9 180-199 164 164 164 275 275 275 F F F 10 200-219 162 162 162 272 272 272 F F F 11 220-239 164 164 164 275 275 275 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 165 165 165 276 276 276 F F F 15 300-319 165 165 165 276 276 276 F F F 16 320-339 162 162 162 272 272 272 F F F 17 340-359 164 164 164 275 275 275 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 165 165 165 276 276 276 F F F 20 400-419 161 161 161 271 271 271 F F F 21 420-439 163 163 163 274 274 274 F F F 22 440-459 165 165 165 276 276 276 F F F 23 460-479 161 161 161 271 271 271 F F F 24 480-499 164 164 164 275 275 275 F F F 25 500-519 164 164 164 275 275 275 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 165, Max flush: 276 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:16.998443306Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:17.040553786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.040940632Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.071637444Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072194064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072236703Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072254332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072272131Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} {"msg":"Crucible c4a2c2d5-1ad5-42d8-b749-c2f35465a35c has session id: 07e04920-e54e-4a4f-9cad-89c7a7cd6cea","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072763613Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Upstairs opts: Upstairs UUID: c4a2c2d5-1ad5-42d8-b749-c2f35465a35c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072806722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Crucible stats registered with UUID: c4a2c2d5-1ad5-42d8-b749-c2f35465a35c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.072829821Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102476781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.10250828Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} {"msg":"c4a2c2d5-1ad5-42d8-b749-c2f35465a35c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102543289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102562408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102588927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102606416Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102623136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102639365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102657074Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102722912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102758021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.1027889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_connection connected from Ok(127.0.0.1:42102)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102829158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"io task","client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_connection connected from Ok(127.0.0.1:59075)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.102859927Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"io task","client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_connection connected from Ok(127.0.0.1:37943)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.103075539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"io task","client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.103583031Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.1036114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.103628109Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.103642029Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.103655508Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.103677977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104267536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104399502Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104423911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104814827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104845426Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104861365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.104964991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.10498564Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.10500284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105015949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105029689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[0]R flush_numbers[0..12]: [275, 261, 259, 275, 272, 276, 274, 273, 276, 275, 272, 275]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105044208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[0]R generation[0..12]: [164, 155, 153, 164, 162, 165, 163, 162, 165, 164, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105058328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105072507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[1]R flush_numbers[0..12]: [275, 261, 259, 275, 272, 276, 274, 273, 276, 275, 272, 275]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105086297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[1]R generation[0..12]: [164, 155, 153, 164, 162, 165, 163, 162, 165, 164, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105100226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105114136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[2]R flush_numbers[0..12]: [275, 261, 259, 275, 272, 276, 274, 273, 276, 275, 272, 275]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105127855Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[2]R generation[0..12]: [164, 155, 153, 164, 162, 165, 163, 162, 165, 164, 162, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105141665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105155634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105168844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105182223Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Next flush: 277","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105195273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105218002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"":"downstairs","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105231812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105244041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"0","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105257251Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"1","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.10527043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"client":"2","":"downstairs","session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.10528371Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105296169Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"c4a2c2d5-1ad5-42d8-b749-c2f35465a35c is now active with session: 07e04920-e54e-4a4f-9cad-89c7a7cd6cea","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105309209Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105321348Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846,"session_id":"07e04920-e54e-4a4f-9cad-89c7a7cd6cea"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:17.105333708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 4 len 4096 data: 10 03/30 Read block 157 len 28672 04/30 Write block 199 len 12288 data: 7 7 8 05/30 Flush 06/30 Write block 310 len 12288 data: 10 9 7 07/30 Write block 56 len 24576 data: 12 12 14 13 15 11 08/30 Read block 536 len 28672 09/30 Read block 334 len 4096 10/30 Flush 11/30 Write block 509 len 32768 data: 12 16 17 16 13 15 12 12 12/30 Read block 531 len 28672 13/30 Read block 213 len 4096 14/30 Flush 15/30 Write block 451 len 8192 data: 13 14 16/30 Read block 468 len 4096 17/30 Read block 488 len 8192 18/30 Write block 445 len 16384 data: 7 10 12 11 19/30 Flush 20/30 Write block 406 len 32768 data: 7 8 10 13 14 13 13 13 21/30 Read block 189 len 8192 22/30 Read block 585 len 36864 23/30 Read block 147 len 4096 24/30 Read block 296 len 32768 25/30 Read block 475 len 28672 26/30 Read block 164 len 28672 27/30 Read block 505 len 12288 28/30 Read block 367 len 12288 29/30 Read block 337 len 12288 30/30 Write block 487 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:11 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1019 Acked Write 8 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Read 9 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 8 Done Done Done false 1024 Acked Read 7 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Read 3 Done Done Done false 1028 Acked Read 3 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 10 10 10 30 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1018 1018 1018 Downstairs last five completed: 1018 1017 1016 1015 1014 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:17.129761026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1846} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 166 166 166 279 279 279 F F F 3 060-079 166 166 166 279 279 279 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 165 165 165 276 276 276 F F F 6 120-139 163 163 163 274 274 274 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 165 165 165 276 276 276 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 164 164 164 275 275 275 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 165 165 165 276 276 276 F F F 15 300-319 166 166 166 279 279 279 F F F 16 320-339 162 162 162 272 272 272 F F F 17 340-359 164 164 164 275 275 275 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 165 165 165 276 276 276 F F F 20 400-419 161 161 161 271 271 271 T T T 21 420-439 163 163 163 274 274 274 F F F 22 440-459 166 166 166 281 281 281 F F F 23 460-479 161 161 161 271 271 271 F F F 24 480-499 164 164 164 275 275 275 T T T 25 500-519 166 166 166 280 280 280 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 166, Max flush: 281 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 166 166 166 279 279 279 F F F 3 060-079 166 166 166 279 279 279 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 165 165 165 276 276 276 F F F 6 120-139 163 163 163 274 274 274 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 165 165 165 276 276 276 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 164 164 164 275 275 275 F F F 12 240-259 161 161 161 271 271 271 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 165 165 165 276 276 276 F F F 15 300-319 166 166 166 279 279 279 F F F 16 320-339 162 162 162 272 272 272 F F F 17 340-359 164 164 164 275 275 275 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 165 165 165 276 276 276 F F F 20 400-419 167 167 167 282 282 282 F F F 21 420-439 163 163 163 274 274 274 F F F 22 440-459 166 166 166 281 281 281 F F F 23 460-479 161 161 161 271 271 271 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 166 166 166 280 280 280 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 158 158 158 265 265 265 F F F 28 560-579 159 159 159 266 266 266 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 167, Max flush: 282 Begin loop 84 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.441877174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:21.486411447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.486782893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.518060384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.518647933Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.518684112Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.518698551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.518713231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} {"msg":"Crucible 3de2b7c0-ff4d-447c-aabf-796f741cc5ef has session id: 2122475c-7904-4781-8c98-d479cf76c10f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.519168544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Upstairs opts: Upstairs UUID: 3de2b7c0-ff4d-447c-aabf-796f741cc5ef, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.519202623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Crucible stats registered with UUID: 3de2b7c0-ff4d-447c-aabf-796f741cc5ef","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.519219133Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549657614Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549703682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} {"msg":"3de2b7c0-ff4d-447c-aabf-796f741cc5ef active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549738951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.54977021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549817738Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549855087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549890605Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549924224Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549958313Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.549990332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550017871Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550053569Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33346)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550081959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"io task","client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33063)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550108818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"io task","client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_connection connected from Ok(127.0.0.1:45824)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550221163Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"io task","client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550819122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.55085997Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.55087979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550952557Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.550984346Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.551016755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.551532546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.551563595Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.551579265Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552101486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552130674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552145944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552159064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552172843Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552186983Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552199722Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552216391Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[0]R flush_numbers[0..12]: [278, 261, 279, 279, 272, 276, 274, 273, 276, 278, 278, 275]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552230881Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[0]R generation[0..12]: [166, 155, 166, 166, 162, 165, 163, 162, 165, 166, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.55224525Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.55225972Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[1]R flush_numbers[0..12]: [278, 261, 279, 279, 272, 276, 274, 273, 276, 278, 278, 275]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552273659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[1]R generation[0..12]: [166, 155, 166, 166, 162, 165, 163, 162, 165, 166, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552287509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552301798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[2]R flush_numbers[0..12]: [278, 261, 279, 279, 272, 276, 274, 273, 276, 278, 278, 275]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552315738Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[2]R generation[0..12]: [166, 155, 166, 166, 162, 165, 163, 162, 165, 166, 166, 164]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552329547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552343467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552356576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552369926Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Next flush: 283","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552392685Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552405815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"":"downstairs","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552420754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552433114Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"0","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552446373Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"1","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552459403Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"client":"2","":"downstairs","session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552472622Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552485102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"3de2b7c0-ff4d-447c-aabf-796f741cc5ef is now active with session: 2122475c-7904-4781-8c98-d479cf76c10f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552497831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.552509971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854,"session_id":"2122475c-7904-4781-8c98-d479cf76c10f"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:21.55252233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 4096 data: 12 02/30 Read block 245 len 24576 03/30 Write block 121 len 32768 data: 14 13 13 14 16 19 20 19 04/30 Read block 23 len 24576 05/30 Write block 344 len 20480 data: 17 16 14 14 17 06/30 Read block 158 len 12288 07/30 Write block 239 len 12288 data: 11 11 8 08/30 Write block 558 len 20480 data: 7 5 7 6 6 09/30 Read block 548 len 40960 10/30 Read block 147 len 24576 11/30 Write block 422 len 28672 data: 11 15 10 13 11 12 11 12/30 Write block 299 len 28672 data: 13 9 9 9 11 9 10 13/30 Read block 139 len 4096 14/30 Write block 380 len 40960 data: 16 17 16 16 15 16 13 10 12 11 15/30 Read block 275 len 28672 16/30 Flush 17/30 Flush 18/30 Write block 401 len 32768 data: 13 11 6 6 7 8 9 11 19/30 Read block 467 len 16384 20/30 Read block 352 len 8192 21/30 Write block 316 len 36864 data: 10 12 11 11 11 14 14 12 11 22/30 Read block 532 len 20480 23/30 Read block 468 len 4096 24/30 Read block 434 len 16384 25/30 Read block 332 len 4096 26/30 Read block 451 len 4096 27/30 Read block 309 len 12288 28/30 Write block 345 len 28672 data: 17 15 15 18 17 12 10 29/30 Read block 547 len 4096 30/30 Write block 467 len 20480 data: 15 13 13 12 12 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:13 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1017 Acked Write 8 Done Done Done false 1018 Acked Read 4 Done Done Done false 1019 Acked Read 2 Done Done Done false 1020 Acked Write 9 Done Done Done false 1021 Acked Read 5 Done Done Done false 1022 Acked Read 1 Done Done Done false 1023 Acked Read 4 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 7 Done Sent Sent false 1028 Acked Read 1 Done Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 3 3 7 Done 12 10 10 32 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1016 1016 1016 Downstairs last five completed: 1016 1015 1014 1013 1012 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:21.576547753Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1854} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 166 166 166 279 279 279 F F F 3 060-079 166 166 166 279 279 279 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 165 165 165 276 276 276 F F F 6 120-139 168 168 168 283 283 283 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 168 168 168 283 283 283 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 168 168 168 283 283 283 F F F 12 240-259 168 168 168 283 283 283 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 168 168 168 283 283 283 F F F 15 300-319 168 168 168 283 283 283 T T T 16 320-339 162 162 162 272 272 272 T T T 17 340-359 168 168 168 283 283 283 T T T 18 360-379 164 164 164 275 275 275 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 167 167 167 282 282 282 T T T 21 420-439 168 168 168 283 283 283 F F F 22 440-459 166 166 166 281 281 281 F F F 23 460-479 161 161 161 271 271 271 T T T 24 480-499 167 167 167 282 282 282 F F F 25 500-519 166 166 166 280 280 280 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 168 168 168 283 283 283 F F F 28 560-579 168 168 168 283 283 283 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 168, Max flush: 283 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 166 166 166 279 279 279 F F F 3 060-079 166 166 166 279 279 279 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 165 165 165 276 276 276 F F F 6 120-139 168 168 168 283 283 283 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 168 168 168 283 283 283 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 168 168 168 283 283 283 F F F 12 240-259 168 168 168 283 283 283 F F F 13 260-279 162 162 162 273 273 273 F F F 14 280-299 168 168 168 283 283 283 F F F 15 300-319 169 169 169 284 284 284 F F F 16 320-339 169 169 169 284 284 284 F F F 17 340-359 169 169 169 284 284 284 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 169 169 169 284 284 284 F F F 21 420-439 168 168 168 283 283 283 F F F 22 440-459 166 166 166 281 281 281 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 166 166 166 280 280 280 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 168 168 168 283 283 283 F F F 28 560-579 168 168 168 283 283 283 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 169, Max flush: 284 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.91805066Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:25.960505547Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.960895723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.991532437Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.992062098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.992098976Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.992113946Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.992128575Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} {"msg":"Crucible 3b0beade-51e9-4147-941d-d5c742fe80ee has session id: 3d6f2244-8fd3-4136-aad4-bbe145795d88","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.99256043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Upstairs opts: Upstairs UUID: 3b0beade-51e9-4147-941d-d5c742fe80ee, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.992599228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Crucible stats registered with UUID: 3b0beade-51e9-4147-941d-d5c742fe80ee","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:25.992623268Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022303996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022350495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} {"msg":"3b0beade-51e9-4147-941d-d5c742fe80ee active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022386573Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022418462Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.02246733Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022504179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022536798Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022566137Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022597955Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022631354Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022667773Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022703042Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_connection connected from Ok(127.0.0.1:39797)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.02274033Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"io task","client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_connection connected from Ok(127.0.0.1:51100)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022941763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"io task","client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_connection connected from Ok(127.0.0.1:40368)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.022994671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"io task","client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.023538661Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.02357195Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.023632858Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.023708355Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.023733034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.023747514Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024175538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024204308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024345043Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024793756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024821845Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024836755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024849384Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024863204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024876513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024888883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024905812Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[0]R flush_numbers[0..12]: [278, 261, 279, 279, 272, 276, 283, 273, 283, 278, 278, 283]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024920442Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[0]R generation[0..12]: [166, 155, 166, 166, 162, 165, 168, 162, 168, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024934531Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024948671Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[1]R flush_numbers[0..12]: [278, 261, 279, 279, 272, 276, 283, 273, 283, 278, 278, 283]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.0249627Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[1]R generation[0..12]: [166, 155, 166, 166, 162, 165, 168, 162, 168, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.02497663Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.024990959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[2]R flush_numbers[0..12]: [278, 261, 279, 279, 272, 276, 283, 273, 283, 278, 278, 283]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025005259Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[2]R generation[0..12]: [166, 155, 166, 166, 162, 165, 168, 162, 168, 166, 166, 168]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025019518Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025033567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:09:26.025046667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025066946Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025080206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025105415Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"":"downstairs","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025122814Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025135264Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"0","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025148723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"1","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025162053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"client":"2","":"downstairs","session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025175553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025188212Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"3b0beade-51e9-4147-941d-d5c742fe80ee is now active with session: 3d6f2244-8fd3-4136-aad4-bbe145795d88","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025201101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025213171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862,"session_id":"3d6f2244-8fd3-4136-aad4-bbe145795d88"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:26.025225811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 41 len 8192 data: 13 10 02/30 Write block 570 len 24576 data: 18 18 14 13 11 8 03/30 Read block 570 len 16384 04/30 Write block 104 len 24576 data: 15 16 17 15 15 15 05/30 Write block 276 len 24576 data: 14 14 13 13 14 14 06/30 Write block 571 len 4096 data: 19 07/30 Read block 392 len 32768 08/30 Read block 9 len 8192 09/30 Read block 115 len 8192 10/30 Flush 11/30 Write block 121 len 32768 data: 15 14 14 15 17 20 21 20 12/30 Read block 574 len 28672 13/30 Read block 189 len 12288 14/30 Read block 458 len 24576 15/30 Read block 79 len 32768 16/30 Flush 17/30 Write block 344 len 12288 data: 18 18 16 18/30 Write block 50 len 12288 data: 10 8 10 19/30 Read block 182 len 36864 20/30 Flush 21/30 Write block 515 len 16384 data: 13 13 13 13 22/30 Flush 23/30 Write block 246 len 20480 data: 14 12 14 15 15 24/30 Read block 151 len 28672 25/30 Flush 26/30 Write block 231 len 12288 data: 10 8 8 27/30 Read block 21 len 16384 28/30 Write block 287 len 36864 data: 21 19 15 12 14 10 12 10 13 29/30 Read block 205 len 20480 30/30 Read block 438 len 4096 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 3 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Write 9 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Read 1 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 5 5 5 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:26.051615068Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1862} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 170 170 170 287 287 287 F F F 3 060-079 166 166 166 279 279 279 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 170 170 170 285 285 285 F F F 6 120-139 170 170 170 286 286 286 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 168 168 168 283 283 283 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 168 168 168 283 283 283 T T T 12 240-259 170 170 170 289 289 289 F F F 13 260-279 170 170 170 285 285 285 F F F 14 280-299 170 170 170 285 285 285 T T T 15 300-319 169 169 169 284 284 284 F F F 16 320-339 169 169 169 284 284 284 F F F 17 340-359 170 170 170 287 287 287 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 169 169 169 284 284 284 F F F 21 420-439 168 168 168 283 283 283 F F F 22 440-459 166 166 166 281 281 281 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 168 168 168 283 283 283 F F F 28 560-579 170 170 170 285 285 285 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 170, Max flush: 289 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 170 170 170 287 287 287 F F F 3 060-079 166 166 166 279 279 279 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 170 170 170 285 285 285 F F F 6 120-139 170 170 170 286 286 286 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 168 168 168 283 283 283 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 171 171 171 290 290 290 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 170 170 170 285 285 285 F F F 14 280-299 171 171 171 290 290 290 F F F 15 300-319 169 169 169 284 284 284 F F F 16 320-339 169 169 169 284 284 284 F F F 17 340-359 170 170 170 287 287 287 F F F 18 360-379 164 164 164 275 275 275 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 169 169 169 284 284 284 F F F 21 420-439 168 168 168 283 283 283 F F F 22 440-459 166 166 166 281 281 281 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 168 168 168 283 283 283 F F F 28 560-579 170 170 170 285 285 285 F F F 29 580-599 159 159 159 266 266 266 F F F Max gen: 171, Max flush: 290 Begin loop 86 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.355871649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.40016763Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.400832996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.400870625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.400885634Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.400900324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} {"msg":"Crucible 102e3c34-3358-4663-baf7-637dc25be87a has session id: 2e071887-b0b0-4f00-bec5-f44b15e3fb55","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.401339938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Upstairs opts: Upstairs UUID: 102e3c34-3358-4663-baf7-637dc25be87a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.401384726Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Crucible stats registered with UUID: 102e3c34-3358-4663-baf7-637dc25be87a","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.401410965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431281027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431313876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} {"msg":"102e3c34-3358-4663-baf7-637dc25be87a active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431386623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431409972Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431438811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.43146853Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431502279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431534678Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431558697Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431575696Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431593006Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431615495Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_connection connected from Ok(127.0.0.1:34490)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431649584Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"io task","client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_connection connected from Ok(127.0.0.1:60402)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431851236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"io task","client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_connection connected from Ok(127.0.0.1:37785)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.431889105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"io task","client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.432348878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.432380707Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.432396467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.432410666Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.432424536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.432442815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433044643Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433088542Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.43312683Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433587674Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433617793Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433633652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433706889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433727709Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433741438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433753898Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433767517Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[0]R flush_numbers[0..12]: [278, 261, 287, 279, 272, 285, 286, 273, 283, 278, 278, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433787307Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[0]R generation[0..12]: [166, 155, 170, 166, 162, 170, 170, 162, 168, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433801756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433815755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[1]R flush_numbers[0..12]: [278, 261, 287, 279, 272, 285, 286, 273, 283, 278, 278, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433834775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[1]R generation[0..12]: [166, 155, 170, 166, 162, 170, 170, 162, 168, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433849034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433863294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[2]R flush_numbers[0..12]: [278, 261, 287, 279, 272, 285, 286, 273, 283, 278, 278, 290]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433877083Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[2]R generation[0..12]: [166, 155, 170, 166, 162, 170, 170, 162, 168, 166, 166, 171]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433890993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433904832Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433918022Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433931691Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Next flush: 291","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.433944721Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.43395788Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"":"downstairs","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.4339715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:09:30.433983809Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"0","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434017048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"1","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434030268Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"client":"2","":"downstairs","session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434043487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434058947Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"102e3c34-3358-4663-baf7-637dc25be87a is now active with session: 2e071887-b0b0-4f00-bec5-f44b15e3fb55","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434072446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434084466Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870,"session_id":"2e071887-b0b0-4f00-bec5-f44b15e3fb55"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:30.434096745Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 267 len 32768 data: 13 11 12 14 16 17 16 15 02/30 Read block 331 len 16384 03/30 Read block 416 len 16384 04/30 Flush 05/30 Write block 426 len 36864 data: 12 13 12 13 12 10 13 13 13 06/30 Flush 07/30 Write block 68 len 36864 data: 9 10 11 12 10 9 11 12 14 08/30 Read block 132 len 4096 09/30 Read block 531 len 24576 10/30 Write block 354 len 32768 data: 12 13 13 14 14 15 18 17 11/30 Write block 409 len 40960 data: 14 15 14 14 14 11 9 10 8 10 12/30 Write block 236 len 4096 data: 6 13/30 Write block 286 len 8192 data: 21 22 14/30 Write block 359 len 12288 data: 16 19 18 15/30 Read block 199 len 8192 16/30 Read block 184 len 16384 17/30 Read block 409 len 4096 18/30 Read block 31 len 12288 19/30 Read block 482 len 24576 20/30 Write block 573 len 20480 data: 14 12 9 7 10 21/30 Flush 22/30 Write block 169 len 24576 data: 11 11 11 9 8 8 23/30 Write block 593 len 16384 data: 14 14 11 9 24/30 Write block 403 len 40960 data: 7 7 8 9 10 12 15 16 15 15 25/30 Write block 451 len 8192 data: 14 15 26/30 Read block 386 len 20480 27/30 Read block 220 len 8192 28/30 Read block 273 len 24576 29/30 Read block 201 len 32768 30/30 Write block 566 len 12288 data: 13 17 16 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Write 4 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Write 2 Done Done Done false 1025 Acked Read 5 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:30.457295478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1870} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 170 170 170 287 287 287 F F F 3 060-079 172 172 172 293 293 293 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 170 170 170 285 285 285 F F F 6 120-139 170 170 170 286 286 286 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 168 168 168 283 283 283 T T T 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 172 172 172 293 293 293 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 172 172 172 291 291 291 F F F 14 280-299 172 172 172 293 293 293 F F F 15 300-319 169 169 169 284 284 284 F F F 16 320-339 169 169 169 284 284 284 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 172 172 172 293 293 293 T T T 21 420-439 172 172 172 292 292 292 F F F 22 440-459 166 166 166 281 281 281 T T T 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 168 168 168 283 283 283 F F F 28 560-579 172 172 172 293 293 293 T T T 29 580-599 159 159 159 266 266 266 T T T Max gen: 172, Max flush: 293 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 170 170 170 287 287 287 F F F 3 060-079 172 172 172 293 293 293 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 170 170 170 285 285 285 F F F 6 120-139 170 170 170 286 286 286 F F F 7 140-159 162 162 162 273 273 273 F F F 8 160-179 173 173 173 294 294 294 F F F 9 180-199 166 166 166 278 278 278 F F F 10 200-219 166 166 166 278 278 278 F F F 11 220-239 172 172 172 293 293 293 F F F 12 240-259 170 170 170 289 289 289 F F F 13 260-279 172 172 172 291 291 291 F F F 14 280-299 172 172 172 293 293 293 F F F 15 300-319 169 169 169 284 284 284 F F F 16 320-339 169 169 169 284 284 284 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 173 173 173 294 294 294 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 168 168 168 283 283 283 F F F 28 560-579 173 173 173 294 294 294 F F F 29 580-599 173 173 173 294 294 294 F F F Max gen: 173, Max flush: 294 Begin loop 87 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.772759223Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.81744419Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818127736Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818177164Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818194153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818209053Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} {"msg":"Crucible 00829276-9043-4d5c-a667-49eed0543492 has session id: f45a1f4e-3224-4bea-9965-788c80f9f964","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818620228Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Upstairs opts: Upstairs UUID: 00829276-9043-4d5c-a667-49eed0543492, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818657686Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Crucible stats registered with UUID: 00829276-9043-4d5c-a667-49eed0543492","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.818674076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849145026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849190294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} {"msg":"00829276-9043-4d5c-a667-49eed0543492 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849220523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849252192Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.84929977Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849336619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849370608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849402827Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849438035Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849472144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849549911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.84959142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_connection connected from Ok(127.0.0.1:33464)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849623979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"io task","client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_connection connected from Ok(127.0.0.1:46321)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849706665Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"io task","client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_connection connected from Ok(127.0.0.1:47542)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.849753664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"io task","client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.850319213Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.850355122Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.850384041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.85041429Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.850446069Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.850473768Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851042667Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851074756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851108825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851646226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851687414Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851716743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851738432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851763171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.85178716Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851810649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851834979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[0]R flush_numbers[0..12]: [278, 261, 287, 293, 272, 285, 286, 273, 294, 278, 278, 293]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851858758Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[0]R generation[0..12]: [166, 155, 170, 172, 162, 170, 170, 162, 173, 166, 166, 172]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851884857Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851912826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[1]R flush_numbers[0..12]: [278, 261, 287, 293, 272, 285, 286, 273, 294, 278, 278, 293]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851939935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[1]R generation[0..12]: [166, 155, 170, 172, 162, 170, 170, 162, 173, 166, 166, 172]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851965534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.851995113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Wait for a query_work_queue command to finish before sending IO [2]R flush_numbers[0..12]: [278, 261, 287, 293, 272, 285, 286, 273, 294, 278, 278, 293]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852038441Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[2]R generation[0..12]: [166, 155, 170, 172, 162, 170, 170, 162, 173, 166, 166, 172]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.85207077Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852101929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852130988Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852159727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Next flush: 295","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:09:34.852187736Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852215655Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"":"downstairs","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852244364Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852266073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"0","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852290942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"1","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852304912Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"client":"2","":"downstairs","session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852318521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852331231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"00829276-9043-4d5c-a667-49eed0543492 is now active with session: f45a1f4e-3224-4bea-9965-788c80f9f964","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.85234406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.85235908Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878,"session_id":"f45a1f4e-3224-4bea-9965-788c80f9f964"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:34.852371539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 16384 data: 16 18 22 20 02/30 Write block 556 len 16384 data: 10 9 8 6 03/30 Write block 233 len 36864 data: 9 5 5 7 8 8 12 12 9 04/30 Read block 472 len 20480 05/30 Write block 258 len 32768 data: 13 10 10 8 8 7 8 10 06/30 Write block 195 len 24576 data: 12 11 11 7 8 8 07/30 Read block 142 len 36864 08/30 Write block 413 len 28672 data: 15 12 10 11 9 11 10 09/30 Write block 558 len 12288 data: 9 7 8 10/30 Read block 104 len 28672 11/30 Write block 154 len 20480 data: 9 11 12 11 9 12/30 Read block 109 len 20480 13/30 Read block 283 len 12288 14/30 Write block 143 len 24576 data: 13 15 21 22 23 16 15/30 Read block 99 len 8192 16/30 Write block 338 len 8192 data: 17 12 17/30 Write block 128 len 28672 data: 21 16 16 16 18 13 17 18/30 Read block 376 len 32768 19/30 Read block 507 len 20480 20/30 Read block 459 len 36864 21/30 Flush 22/30 Write block 592 len 20480 data: 13 15 15 12 10 23/30 Write block 312 len 20480 data: 8 7 7 6 11 24/30 Read block 230 len 28672 25/30 Read block 28 len 36864 26/30 Read block 452 len 12288 27/30 Read block 334 len 24576 28/30 Read block 313 len 32768 29/30 Write block 44 len 20480 data: 8 9 9 9 8 30/30 Read block 316 len 36864 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 9 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Read 8 Done Done Done false 1028 Acked Write 5 Done Done Done false 1029 Acked Read 9 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 0 1 Done 8 9 9 26 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:34.87394919Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1878} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 170 170 170 287 287 287 T T T 3 060-079 172 172 172 293 293 293 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 170 170 170 285 285 285 F F F 6 120-139 174 174 174 295 295 295 F F F 7 140-159 174 174 174 295 295 295 F F F 8 160-179 173 173 173 294 294 294 F F F 9 180-199 174 174 174 295 295 295 F F F 10 200-219 174 174 174 295 295 295 F F F 11 220-239 174 174 174 295 295 295 F F F 12 240-259 174 174 174 295 295 295 F F F 13 260-279 174 174 174 295 295 295 F F F 14 280-299 172 172 172 293 293 293 F F F 15 300-319 169 169 169 284 284 284 T T T 16 320-339 174 174 174 295 295 295 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 174 174 174 295 295 295 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 173 173 173 294 294 294 T T T Max gen: 174, Max flush: 295 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 155 155 155 261 261 261 F F F 2 040-059 175 175 175 296 296 296 F F F 3 060-079 172 172 172 293 293 293 F F F 4 080-099 162 162 162 272 272 272 F F F 5 100-119 170 170 170 285 285 285 F F F 6 120-139 174 174 174 295 295 295 F F F 7 140-159 174 174 174 295 295 295 F F F 8 160-179 173 173 173 294 294 294 F F F 9 180-199 174 174 174 295 295 295 F F F 10 200-219 174 174 174 295 295 295 F F F 11 220-239 174 174 174 295 295 295 F F F 12 240-259 174 174 174 295 295 295 F F F 13 260-279 174 174 174 295 295 295 F F F 14 280-299 172 172 172 293 293 293 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 174 174 174 295 295 295 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 168 168 168 283 283 283 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 167 167 167 282 282 282 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 174 174 174 295 295 295 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 175 175 175 296 296 296 F F F Max gen: 175, Max flush: 296 Begin loop 88 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.163060318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.206466371Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.207076529Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.207120208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.207142877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.207161406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} {"msg":"Crucible 28f0c5c7-03bc-4a0b-a684-75d55a7cf6fa has session id: 816f17c0-c659-4cf6-b3a4-ae4e069838dd","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.207558332Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Upstairs opts: Upstairs UUID: 28f0c5c7-03bc-4a0b-a684-75d55a7cf6fa, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.20760787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Crucible stats registered with UUID: 28f0c5c7-03bc-4a0b-a684-75d55a7cf6fa","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.207631239Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237409544Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237442513Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} {"msg":"28f0c5c7-03bc-4a0b-a684-75d55a7cf6fa active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.23751353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.23753548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237562619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237580448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237597287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237611887Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237625497Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237638996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237652836Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237671065Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:36850)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.237961854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"io task","client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:33099)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238010652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"io task","client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_connection connected from Ok(127.0.0.1:44997)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238044191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"io task","client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238553593Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238591331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238607471Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238621541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.23863877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.238665889Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.23919753Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239227628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239243558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239767319Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239796738Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239815007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239828317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239841946Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239855166Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239867325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239880795Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[0]R flush_numbers[0..12]: [278, 261, 296, 293, 272, 285, 295, 295, 294, 295, 295, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239904964Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[0]R generation[0..12]: [166, 155, 175, 172, 162, 170, 174, 174, 173, 174, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239920063Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239934253Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[1]R flush_numbers[0..12]: [278, 261, 296, 293, 272, 285, 295, 295, 294, 295, 295, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239948273Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[1]R generation[0..12]: [166, 155, 175, 172, 162, 170, 174, 174, 173, 174, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239962352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239976541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[2]R flush_numbers[0..12]: [278, 261, 296, 293, 272, 285, 295, 295, 294, 295, 295, 295]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.239990561Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[2]R generation[0..12]: [166, 155, 175, 172, 162, 170, 174, 174, 173, 174, 174, 174]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240004331Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.24002048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Max found gen is 176","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.24003411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240047599Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Next flush: 297","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240060689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240073538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"":"downstairs","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg"Wait for a query_work_queue command to finish before sending IO :"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240098377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240110717Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"0","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240123846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"1","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240136826Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"client":"2","":"downstairs","session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240150085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240162275Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"28f0c5c7-03bc-4a0b-a684-75d55a7cf6fa is now active with session: 816f17c0-c659-4cf6-b3a4-ae4e069838dd","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240175084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240187174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886,"session_id":"816f17c0-c659-4cf6-b3a4-ae4e069838dd"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:39.240199524Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 28672 data: 14 14 11 9 9 10 10 02/30 Write block 24 len 16384 data: 12 17 16 14 03/30 Read block 517 len 4096 04/30 Read block 351 len 28672 05/30 Write block 487 len 28672 data: 15 13 11 10 11 12 11 06/30 Read block 123 len 8192 07/30 Write block 232 len 16384 data: 9 10 6 6 08/30 Read block 129 len 20480 09/30 Read block 132 len 16384 10/30 Read block 446 len 12288 11/30 Read block 208 len 12288 12/30 Read block 26 len 4096 13/30 Write block 191 len 16384 data: 11 11 13 15 14/30 Write block 382 len 24576 data: 17 17 16 17 14 11 15/30 Read block 462 len 4096 16/30 Read block 242 len 24576 17/30 Read block 346 len 32768 18/30 Write block 69 len 4096 data: 11 19/30 Write block 278 len 32768 data: 14 14 15 15 16 21 19 20 20/30 Read block 544 len 8192 21/30 Read block 294 len 16384 22/30 Write block 552 len 4096 data: 19 23/30 Flush 24/30 Write block 43 len 16384 data: 10 10 11 11 25/30 Write block 98 len 36864 data: 9 6 8 11 13 14 16 17 18 26/30 Write block 212 len 36864 data: 8 12 8 11 18 17 15 14 10 27/30 Write block 209 len 24576 data: 13 12 10 9 13 9 28/30 Write block 116 len 40960 data: 17 17 17 17 17 16 15 15 16 18 29/30 Write block 158 len 36864 data: 10 10 11 9 12 12 13 15 16 30/30 Read block 236 len 40960 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 4 Done Done Done false 1024 Acked Write 9 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Write 6 Done Done Done false 1027 Acked Write 10 Done Done Done false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 10 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 2 3 Done 6 7 5 18 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:39.261796204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1886} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 176 176 176 297 297 297 F F F 2 040-059 176 176 176 297 297 297 T T T 3 060-079 176 176 176 297 297 297 F F F 4 080-099 162 162 162 272 272 272 T T T 5 100-119 170 170 170 285 285 285 T T T 6 120-139 174 174 174 295 295 295 T T T 7 140-159 174 174 174 295 295 295 T T T 8 160-179 173 173 173 294 294 294 T T T 9 180-199 176 176 176 297 297 297 F F F 10 200-219 174 174 174 295 295 295 T T T 11 220-239 176 176 176 297 297 297 T T T 12 240-259 174 174 174 295 295 295 F F F 13 260-279 176 176 176 297 297 297 F F F 14 280-299 176 176 176 297 297 297 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 174 174 174 295 295 295 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 176 176 176 297 297 297 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 175 175 175 296 296 296 F F F Max gen: 176, Max flush: 297 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 176 176 176 297 297 297 F F F 2 040-059 177 177 177 298 298 298 F F F 3 060-079 176 176 176 297 297 297 F F F 4 080-099 177 177 177 298 298 298 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 177 177 177 298 298 298 F F F 7 140-159 177 177 177 298 298 298 F F F 8 160-179 177 177 177 298 298 298 F F F 9 180-199 176 176 176 297 297 297 F F F 10 200-219 177 177 177 298 298 298 F F F 11 220-239 177 177 177 298 298 298 F F F 12 240-259 174 174 174 295 295 295 F F F 13 260-279 176 176 176 297 297 297 F F F 14 280-299 176 176 176 297 297 297 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 174 174 174 295 295 295 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 172 172 172 292 292 292 F F F 22 440-459 173 173 173 294 294 294 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 158 158 158 265 265 265 F F F 27 540-559 176 176 176 297 297 297 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 175 175 175 296 296 296 F F F Max gen: 177, Max flush: 298 Begin loop 89 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.613286491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.657579792Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658255457Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658317485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658345244Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658375593Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} {"msg":"Crucible b5fe3eb3-557a-4556-99ab-737d2fe6dd79 has session id: de3ad03d-b3db-456f-95de-90bed2abac4e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658802728Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Upstairs opts: Upstairs UUID: b5fe3eb3-557a-4556-99ab-737d2fe6dd79, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658851676Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Crucible stats registered with UUID: b5fe3eb3-557a-4556-99ab-737d2fe6dd79","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.658877445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.68865286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688688089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} {"msg":"b5fe3eb3-557a-4556-99ab-737d2fe6dd79 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688723418Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688742097Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688769656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688789105Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688807575Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688824304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688841153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688866362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688900411Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.68892914Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:49472)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688948899Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"io task","client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52021)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.688975538Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"io task","client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:34299)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.689010437Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"io task","client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.689731681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.68976034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.689894925Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.689936504Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.689963113Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.689990152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.690451785Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.690482404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.690731225Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691031954Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691060283Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691076262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691116731Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.69113486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.69115218Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691250136Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691269186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[0]R flush_numbers[0..12]: [278, 297, 298, 297, 298, 298, 298, 298, 298, 297, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691283425Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[0]R generation[0..12]: [166, 176, 177, 176, 177, 177, 177, 177, 177, 176, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691297554Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691311934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[1]R flush_numbers[0..12]: [278, 297, 298, 297, 298, 298, 298, 298, 298, 297, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691326004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[1]R generation[0..12]: [166, 176, 177, 176, 177, 177, 177, 177, 177, 176, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691339743Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691353992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[2]R flush_numbers[0..12]: [278, 297, 298, 297, 298, 298, 298, 298, 298, 297, 298, 298]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691367762Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[2]R generation[0..12]: [166, 176, 177, 176, 177, 177, 177, 177, 177, 176, 177, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691381582Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691395631Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691408901Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.69142226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.69143816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691451699Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"":"downstairs","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691466148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691478428Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"0","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msgWait for a query_work_queue command to finish before sending IO ":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691512597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"1","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691527056Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"client":"2","":"downstairs","session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691540876Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691553295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"b5fe3eb3-557a-4556-99ab-737d2fe6dd79 is now active with session: de3ad03d-b3db-456f-95de-90bed2abac4e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691566085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691578064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894,"session_id":"de3ad03d-b3db-456f-95de-90bed2abac4e"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:43.691590314Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 82 len 4096 data: 8 02/30 Write block 43 len 36864 data: 11 11 12 12 10 9 7 11 9 03/30 Flush 04/30 Write block 249 len 4096 data: 16 05/30 Write block 430 len 24576 data: 13 11 14 14 14 11 06/30 Read block 17 len 32768 07/30 Flush 08/30 Write block 540 len 8192 data: 9 9 09/30 Write block 206 len 24576 data: 16 13 13 14 13 11 10/30 Flush 11/30 Write block 528 len 12288 data: 9 10 11 12/30 Write block 20 len 4096 data: 14 13/30 Write block 327 len 36864 data: 12 14 14 18 21 20 18 17 17 14/30 Write block 86 len 36864 data: 15 19 20 20 21 17 16 17 16 15/30 Read block 370 len 12288 16/30 Read block 258 len 8192 17/30 Flush 18/30 Write block 152 len 16384 data: 13 10 10 12 19/30 Write block 440 len 40960 data: 8 8 5 5 6 8 11 13 12 14 20/30 Read block 337 len 24576 21/30 Flush 22/30 Write block 74 len 12288 data: 12 13 15 23/30 Read block 214 len 8192 24/30 Flush 25/30 Write block 590 len 16384 data: 19 15 14 16 26/30 Write block 435 len 16384 data: 12 10 10 10 27/30 Read block 196 len 36864 28/30 Read block 577 len 28672 29/30 Read block 88 len 32768 30/30 Read block 180 len 24576 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:6 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1024 Acked Write 4 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Read 7 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 1 2 Done 5 6 5 16 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1023 1023 1023 Downstairs last five completed: 1023 1022 1021 1020 1019 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:43.719179318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1894} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 178 178 178 302 302 302 F F F 2 040-059 178 178 178 299 299 299 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 177 177 177 298 298 298 F F F 7 140-159 178 178 178 303 303 303 F F F 8 160-179 177 177 177 298 298 298 F F F 9 180-199 176 176 176 297 297 297 F F F 10 200-219 178 178 178 301 301 301 F F F 11 220-239 177 177 177 298 298 298 F F F 12 240-259 178 178 178 300 300 300 F F F 13 260-279 176 176 176 297 297 297 F F F 14 280-299 176 176 176 297 297 297 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 178 178 178 302 302 302 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 178 178 178 300 300 300 T T T 22 440-459 178 178 178 303 303 303 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 178 178 178 302 302 302 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 175 175 175 296 296 296 T T T Max gen: 178, Max flush: 304 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 166 166 166 278 278 278 F F F 1 020-039 178 178 178 302 302 302 F F F 2 040-059 178 178 178 299 299 299 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 177 177 177 298 298 298 F F F 7 140-159 178 178 178 303 303 303 F F F 8 160-179 177 177 177 298 298 298 F F F 9 180-199 176 176 176 297 297 297 F F F 10 200-219 178 178 178 301 301 301 F F F 11 220-239 177 177 177 298 298 298 F F F 12 240-259 178 178 178 300 300 300 F F F 13 260-279 176 176 176 297 297 297 F F F 14 280-299 176 176 176 297 297 297 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 178 178 178 302 302 302 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 179 179 179 305 305 305 F F F 22 440-459 178 178 178 303 303 303 F F F 23 460-479 169 169 169 284 284 284 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 170 170 170 288 288 288 F F F 26 520-539 178 178 178 302 302 302 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 179 179 179 305 305 305 F F F Max gen: 179, Max flush: 305 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:47.994766883Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.038116598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.038751026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.038787944Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.038802854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.038817823Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} {"msg":"Crucible b2d49d43-3487-4193-a99c-dcb0e4656187 has session id: 2dc845ca-d4aa-4fc9-a998-49427933d53f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.039277777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Upstairs opts: Upstairs UUID: b2d49d43-3487-4193-a99c-dcb0e4656187, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.039322055Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Crucible stats registered with UUID: b2d49d43-3487-4193-a99c-dcb0e4656187","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.039346494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.068893227Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.068966165Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} {"msg":"b2d49d43-3487-4193-a99c-dcb0e4656187 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.068993614Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069009153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069033093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069047962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069061852Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069075261Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069088771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.06910206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069117179Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069142738Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63614)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069197007Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"io task","client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_connection connected from Ok(127.0.0.1:54086)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069479046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"io task","client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_connection connected from Ok(127.0.0.1:33371)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.069545724Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"io task","client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070019137Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070052356Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070090184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070116303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070131553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070145142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070622825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.07076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.070917825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071198874Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071235463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071262272Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071412436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071441786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071456505Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071518963Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071537962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[0]R flush_numbers[0..12]: [278, 302, 299, 304, 302, 298, 298, 303, 298, 297, 301, 298]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071552591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[0]R generation[0..12]: [166, 178, 178, 178, 178, 177, 177, 178, 177, 176, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071570951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.07158546Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[1]R flush_numbers[0..12]: [278, 302, 299, 304, 302, 298, 298, 303, 298, 297, 301, 298]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.07159963Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[1]R generation[0..12]: [166, 178, 178, 178, 178, 177, 177, 178, 177, 176, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071613409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071627659Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[2]R flush_numbers[0..12]: [278, 302, 299, 304, 302, 298, 298, 303, 298, 297, 301, 298]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071641558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[2]R generation[0..12]: [166, 178, 178, 178, 178, 177, 177, 178, 177, 176, 178, 177]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071655308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071669447Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071682657Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071696056Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Next flush: 306","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071709236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071722435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"":"downstairs","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071735935Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071748335Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"0","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} Wait for a query_work_queue command to finish before sending IO {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071775703Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"1","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071789653Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"client":"2","":"downstairs","session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071803142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071815592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"b2d49d43-3487-4193-a99c-dcb0e4656187 is now active with session: 2dc845ca-d4aa-4fc9-a998-49427933d53f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071828592Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071840731Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902,"session_id":"2dc845ca-d4aa-4fc9-a998-49427933d53f"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:48.071853001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 157 len 28672 data: 12 11 11 12 10 13 13 02/30 Read block 389 len 12288 03/30 Write block 220 len 4096 data: 11 04/30 Read block 558 len 24576 05/30 Read block 288 len 40960 06/30 Read block 442 len 4096 07/30 Read block 204 len 40960 08/30 Write block 17 len 8192 data: 12 11 09/30 Write block 432 len 8192 data: 15 15 10/30 Flush 11/30 Write block 509 len 28672 data: 13 17 18 17 14 16 14 12/30 Write block 188 len 12288 data: 13 15 12 13/30 Read block 407 len 32768 14/30 Read block 176 len 20480 15/30 Write block 459 len 24576 data: 16 13 18 18 18 16 16/30 Read block 277 len 32768 17/30 Read block 343 len 28672 18/30 Write block 277 len 4096 data: 15 19/30 Write block 201 len 8192 data: 9 11 20/30 Read block 300 len 36864 21/30 Read block 205 len 28672 22/30 Write block 421 len 24576 data: 12 12 16 11 14 13 23/30 Flush 24/30 Write block 530 len 28672 data: 12 15 14 12 12 11 10 25/30 Read block 126 len 32768 26/30 Write block 516 len 24576 data: 14 14 14 10 11 13 27/30 Read block 315 len 20480 28/30 Write block 124 len 24576 data: 17 19 21 22 22 17 29/30 Read block 358 len 32768 30/30 Write block 275 len 4096 data: 14 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:7 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1023 Acked Write 7 Done Done Done false 1024 Acked Read 8 Done Done Done false 1025 Acked Write 6 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 8 Done Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 2 3 6 Done 6 5 4 15 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1022 1022 1022 Downstairs last five completed: 1022 1021 1020 1019 1018 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:48.095287015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1902} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 178 178 178 302 302 302 F F F 2 040-059 178 178 178 299 299 299 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 177 177 177 298 298 298 T T T 7 140-159 180 180 180 306 306 306 F F F 8 160-179 180 180 180 306 306 306 F F F 9 180-199 180 180 180 307 307 307 F F F 10 200-219 180 180 180 307 307 307 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 178 178 178 300 300 300 F F F 13 260-279 180 180 180 307 307 307 T T T 14 280-299 176 176 176 297 297 297 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 178 178 178 302 302 302 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 180 180 180 307 307 307 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 180 180 180 307 307 307 T T T 26 520-539 178 178 178 302 302 302 T T T 27 540-559 178 178 178 301 301 301 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 179 179 179 305 305 305 F F F Max gen: 180, Max flush: 307 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 178 178 178 302 302 302 F F F 2 040-059 178 178 178 299 299 299 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 180 180 180 306 306 306 F F F 8 160-179 180 180 180 306 306 306 F F F 9 180-199 180 180 180 307 307 307 F F F 10 200-219 180 180 180 307 307 307 F F F 11 220-239 180 180 180 306 306 306 F F F 12 240-259 178 178 178 300 300 300 F F F 13 260-279 181 181 181 308 308 308 F F F 14 280-299 176 176 176 297 297 297 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 178 178 178 302 302 302 F F F 17 340-359 172 172 172 293 293 293 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 180 180 180 307 307 307 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 181 181 181 308 308 308 F F F 27 540-559 178 178 178 301 301 301 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 179 179 179 305 305 305 F F F Max gen: 181, Max flush: 308 Begin loop 91 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.393336171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:52.436322619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.436713995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.468029254Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.468668191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.46870118Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.468715299Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.468729609Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} {"msg":"Crucible 2a081a5a-9a40-4b56-a4b7-2ce87111cedb has session id: 40671cb1-0ffb-4d01-9c4f-e85405e4515c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.469240371Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Upstairs opts: Upstairs UUID: 2a081a5a-9a40-4b56-a4b7-2ce87111cedb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.469278699Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Crucible stats registered with UUID: 2a081a5a-9a40-4b56-a4b7-2ce87111cedb","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.469295498Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499603835Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499637633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} {"msg":"2a081a5a-9a40-4b56-a4b7-2ce87111cedb active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499687641Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.49971142Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499738199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499756929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499788608Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499822316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499855815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499889494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499924993Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.499957712Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_connection connected from Ok(127.0.0.1:61211)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.49999216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"io task","client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44245)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500024379Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"io task","client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59343)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500231342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"io task","client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500776922Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.5008197Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500851489Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500881298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500911847Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.500946226Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.50137844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.501473897Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.501751297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.501919541Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.501960839Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.501991328Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502019277Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502049356Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502071365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502239999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502276318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[0]R flush_numbers[0..12]: [306, 302, 299, 304, 302, 298, 308, 306, 306, 307, 307, 306]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502302257Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[0]R generation[0..12]: [180, 178, 178, 178, 178, 177, 181, 180, 180, 180, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502327956Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502358285Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[1]R flush_numbers[0..12]: [306, 302, 299, 304, 302, 298, 308, 306, 306, 307, 307, 306]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502384104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[1]R generation[0..12]: [180, 178, 178, 178, 178, 177, 181, 180, 180, 180, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502408893Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502435702Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[2]R flush_numbers[0..12]: [306, 302, 299, 304, 302, 298, 308, 306, 306, 307, 307, 306]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502463781Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[2]R generation[0..12]: [180, 178, 178, 178, 178, 177, 181, 180, 180, 180, 180, 180]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.50248879Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502517949Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Max found gen is 182","v":0,"name":"crucible",Wait for a query_work_queue command to finish before sending IO "level":30,"time":"2024-11-07T14:09:52.502556458Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502588837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Next flush: 309","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502617846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502646025Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"":"downstairs","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502674393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502701483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"0","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2024-11-07T14:09:52.502729871Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"1","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.50277174Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"client":"2","":"downstairs","session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502793149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502806099Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"2a081a5a-9a40-4b56-a4b7-2ce87111cedb is now active with session: 40671cb1-0ffb-4d01-9c4f-e85405e4515c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502819148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502831098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910,"session_id":"40671cb1-0ffb-4d01-9c4f-e85405e4515c"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:52.502843478Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 351 len 20480 data: 11 11 9 13 14 02/30 Read block 532 len 36864 03/30 Read block 499 len 16384 04/30 Read block 51 len 8192 05/30 Flush 06/30 Write block 344 len 40960 data: 19 19 17 16 19 18 13 12 12 10 07/30 Write block 346 len 4096 data: 18 08/30 Read block 552 len 28672 09/30 Read block 228 len 24576 10/30 Read block 49 len 28672 11/30 Write block 242 len 12288 data: 10 11 15 12/30 Write block 226 len 40960 data: 10 12 9 11 11 11 10 11 7 7 13/30 Write block 450 len 24576 data: 16 15 16 19 17 14 14/30 Read block 212 len 4096 15/30 Write block 152 len 8192 data: 14 11 16/30 Write block 39 len 28672 data: 15 15 15 12 12 12 13 17/30 Read block 446 len 36864 18/30 Read block 172 len 16384 19/30 Write block 594 len 24576 data: 16 13 11 9 10 8 20/30 Read block 579 len 8192 21/30 Flush 22/30 Write block 275 len 24576 data: 15 15 16 15 15 16 23/30 Read block 236 len 24576 24/30 Write block 329 len 28672 data: 15 19 22 21 19 18 18 25/30 Read block 157 len 24576 26/30 Write block 580 len 16384 data: 9 8 9 13 27/30 Read block 456 len 20480 28/30 Write block 550 len 32768 data: 18 18 20 21 15 12 11 10 29/30 Read block 378 len 4096 30/30 Write block 589 len 40960 data: 19 20 16 15 17 17 14 12 10 11 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 6 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 7 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 4 Done Done Done false 1026 Acked Read 5 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 1 Done Done Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:52.527084932Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1910} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 182 182 182 310 310 310 F F F 2 040-059 182 182 182 310 310 310 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 182 182 182 310 310 310 F F F 8 160-179 180 180 180 306 306 306 F F F 9 180-199 180 180 180 307 307 307 F F F 10 200-219 180 180 180 307 307 307 F F F 11 220-239 182 182 182 310 310 310 F F F 12 240-259 182 182 182 310 310 310 F F F 13 260-279 181 181 181 308 308 308 T T T 14 280-299 176 176 176 297 297 297 T T T 15 300-319 175 175 175 296 296 296 F F F 16 320-339 178 178 178 302 302 302 T T T 17 340-359 182 182 182 310 310 310 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 182 182 182 310 310 310 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 181 181 181 308 308 308 F F F 27 540-559 178 178 178 301 301 301 T T T 28 560-579 174 174 174 295 295 295 F F F 29 580-599 182 182 182 310 310 310 T T T Max gen: 182, Max flush: 310 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 182 182 182 310 310 310 F F F 2 040-059 182 182 182 310 310 310 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 182 182 182 310 310 310 F F F 8 160-179 180 180 180 306 306 306 F F F 9 180-199 180 180 180 307 307 307 F F F 10 200-219 180 180 180 307 307 307 F F F 11 220-239 182 182 182 310 310 310 F F F 12 240-259 182 182 182 310 310 310 F F F 13 260-279 183 183 183 311 311 311 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 182 182 182 310 310 310 F F F 18 360-379 172 172 172 293 293 293 F F F 19 380-399 176 176 176 297 297 297 F F F 20 400-419 174 174 174 295 295 295 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 182 182 182 310 310 310 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 181 181 181 308 308 308 F F F 27 540-559 183 183 183 311 311 311 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 183 183 183 311 311 311 F F F Max gen: 183, Max flush: 311 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.873872317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:56.915740856Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.916150021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.94695089Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.947552298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.947582367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.947595706Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.947609776Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} {"msg":"Crucible d9cb51fb-813f-483e-afb4-63aeaedaacf6 has session id: fc271994-bf6f-4963-92bb-d426164357ea","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.948074539Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Upstairs opts: Upstairs UUID: d9cb51fb-813f-483e-afb4-63aeaedaacf6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.948116017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Crucible stats registered with UUID: d9cb51fb-813f-483e-afb4-63aeaedaacf6","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.948139187Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978605377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978636296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} {"msg":"d9cb51fb-813f-483e-afb4-63aeaedaacf6 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978712693Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978734182Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978767511Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.97880125Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978834619Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978861258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978892217Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978923995Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.978957144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979180896Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:46509)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979227034Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"io task","client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:64147)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979260353Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"io task","client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_connection connected from Ok(127.0.0.1:40785)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979292512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"io task","client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979859511Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.97990178Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979933799Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979967148Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.979998046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.980032445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.980554486Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.980597755Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.980633374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981176324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981206533Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981222322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981235092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981248721Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981262161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981274431Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.9812878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[0]R flush_numbers[0..12]: [306, 310, 310, 304, 302, 298, 308, 310, 306, 307, 307, 310]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981301819Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[0]R generation[0..12]: [180, 182, 182, 178, 178, 177, 181, 182, 180, 180, 180, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981315939Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981330009Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[1]R flush_numbers[0..12]: [306, 310, 310, 304, 302, 298, 308, 310, 306, 307, 307, 310]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981347878Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[1]R generation[0..12]: [180, 182, 182, 178, 178, 177, 181, 182, 180, 180, 180, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981362467Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981376477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"Wait for a query_work_queue command to finish before sending IO msg":"[2]R flush_numbers[0..12]: [306, 310, 310, 304, 302, 298, 308, 310, 306, 307, 307, 310]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981401506Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[2]R generation[0..12]: [180, 182, 182, 178, 178, 177, 181, 182, 180, 180, 180, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981429445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981456764Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981483173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981509102Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Next flush: 312","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981526971Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981540491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"":"downstairs","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.98155394Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.98156628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"0","":"downstairs","Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981583369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"1","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981597129Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"client":"2","":"downstairs","session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981612998Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981625708Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"d9cb51fb-813f-483e-afb4-63aeaedaacf6 is now active with session: fc271994-bf6f-4963-92bb-d426164357ea","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981638737Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981650837Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918,"session_id":"fc271994-bf6f-4963-92bb-d426164357ea"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:09:56.981663306Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 205 len 32768 data: 15 17 14 14 15 14 12 10 02/30 Read block 265 len 32768 03/30 Read block 88 len 12288 04/30 Read block 497 len 28672 05/30 Read block 125 len 4096 06/30 Read block 68 len 8192 07/30 Read block 78 len 4096 08/30 Write block 540 len 40960 data: 10 10 8 7 11 11 17 19 23 21 09/30 Read block 140 len 32768 10/30 Read block 466 len 36864 11/30 Read block 2 len 36864 12/30 Read block 186 len 32768 13/30 Read block 383 len 8192 14/30 Flush 15/30 Write block 521 len 4096 data: 14 16/30 Read block 219 len 36864 17/30 Read block 476 len 32768 18/30 Read block 64 len 28672 19/30 Read block 131 len 40960 20/30 Read block 343 len 8192 21/30 Write block 154 len 20480 data: 11 13 13 13 12 22/30 Write block 416 len 8192 data: 12 10 23/30 Read block 193 len 12288 24/30 Read block 187 len 28672 25/30 Flush 26/30 Write block 378 len 24576 data: 15 15 17 18 18 18 27/30 Read block 543 len 36864 28/30 Write block 588 len 12288 data: 18 20 21 29/30 Read block 437 len 24576 30/30 Read block 325 len 36864 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 6 Done Done Done false 1026 Acked Read 9 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 6 Done Done Done false 1029 Acked Read 9 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 0 1 Done 5 4 5 14 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:09:57.005235975Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1918} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 182 182 182 310 310 310 F F F 2 040-059 182 182 182 310 310 310 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 180 180 180 306 306 306 F F F 9 180-199 180 180 180 307 307 307 F F F 10 200-219 184 184 184 312 312 312 F F F 11 220-239 182 182 182 310 310 310 F F F 12 240-259 182 182 182 310 310 310 F F F 13 260-279 183 183 183 311 311 311 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 182 182 182 310 310 310 F F F 18 360-379 172 172 172 293 293 293 T T T 19 380-399 176 176 176 297 297 297 T T T 20 400-419 184 184 184 313 313 313 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 182 182 182 310 310 310 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 183 183 183 311 311 311 T T T Max gen: 184, Max flush: 313 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 182 182 182 310 310 310 F F F 2 040-059 182 182 182 310 310 310 F F F 3 060-079 178 178 178 304 304 304 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 184 184 184 313 313 313 F F F 8 160-179 180 180 180 306 306 306 F F F 9 180-199 180 180 180 307 307 307 F F F 10 200-219 184 184 184 312 312 312 F F F 11 220-239 182 182 182 310 310 310 F F F 12 240-259 182 182 182 310 310 310 F F F 13 260-279 183 183 183 311 311 311 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 175 175 175 296 296 296 F F F 16 320-339 183 183 183 311 311 311 F F F 17 340-359 182 182 182 310 310 310 F F F 18 360-379 185 185 185 314 314 314 F F F 19 380-399 185 185 185 314 314 314 F F F 20 400-419 184 184 184 313 313 313 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 182 182 182 310 310 310 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 174 174 174 295 295 295 F F F 29 580-599 185 185 185 314 314 314 F F F Max gen: 185, Max flush: 314 Begin loop 93 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.319111329Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.365488915Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.366127172Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.36616293Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.36618153Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.366197519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} {"msg":"Crucible 928dcec8-5243-47d4-b70c-0f5c48a35fcd has session id: c3aa7ac9-0e51-41da-86d9-7341f3ae8acc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.366694551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Upstairs opts: Upstairs UUID: 928dcec8-5243-47d4-b70c-0f5c48a35fcd, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.3667392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Crucible stats registered with UUID: 928dcec8-5243-47d4-b70c-0f5c48a35fcd","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.366764509Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396694079Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396730317Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} {"msg":"928dcec8-5243-47d4-b70c-0f5c48a35fcd active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396782725Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396804275Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396832443Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396850323Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396867352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396884432Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396901501Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.39692312Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396941419Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.396959149Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_connection connected from Ok(127.0.0.1:38682)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.397129343Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"io task","client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_connection connected from Ok(127.0.0.1:47322)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.397176831Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"io task","client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37468)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.39721545Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"io task","client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.397950623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398018351Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.39804284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398064999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398154076Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398173365Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398664957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398798152Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.398825211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399188769Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399215628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399229917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399369382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399389001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399402551Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399477098Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399496287Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[0]R flush_numbers[0..12]: [306, 310, 310, 304, 302, 298, 308, 313, 306, 307, 312, 310]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399510507Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[0]R generation[0..12]: [180, 182, 182, 178, 178, 177, 181, 184, 180, 180, 184, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399537986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399553405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[1]R flush_numbers[0..12]: [306, 310, 310, 304, 302, 298, 308, 313, 306, 307, 312, 310]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399567345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[1]R generation[0..12]: [180, 182, 182, 178, 178, 177, 181, 184, 180, 180, 184, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399581334Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399595304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[2]R flush_numbers[0..12]: [306, 310, 310, 304, 302, 298, 308, 313, 306, 307, 312, 310]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399609233Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[2]R generation[0..12]: [180, 182, 182, 178, 178, 177, 181, 184, 180, 180, 184, 182]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399626093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399640472Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399653662Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Generation requested: 186 >= found:186","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399666981Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Next flush: 315","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399680171Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.39969349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"":"downstairs","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:10:01.39970703Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399725209Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"0","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399738598Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"1","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399751868Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"client":"2","":"downstairs","session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399765238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399777597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"928dcec8-5243-47d4-b70c-0f5c48a35fcd is now active with session: c3aa7ac9-0e51-41da-86d9-7341f3ae8acc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399790577Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399802786Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926,"session_id":"c3aa7ac9-0e51-41da-86d9-7341f3ae8acc"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:01.399815046Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 447 len 4096 data: 14 02/30 Read block 537 len 24576 03/30 Read block 448 len 4096 04/30 Write block 379 len 12288 data: 16 18 19 05/30 Write block 154 len 4096 data: 12 06/30 Write block 176 len 20480 data: 9 10 10 13 13 07/30 Flush 08/30 Write block 172 len 20480 data: 10 9 9 10 10 09/30 Read block 377 len 4096 10/30 Read block 276 len 40960 11/30 Read block 456 len 4096 12/30 Write block 113 len 12288 data: 20 19 19 13/30 Read block 59 len 24576 14/30 Write block 262 len 28672 data: 9 8 9 11 11 14 12 15/30 Write block 575 len 36864 data: 10 8 11 10 9 10 9 10 14 16/30 Write block 202 len 24576 data: 12 15 16 16 18 15 17/30 Read block 206 len 8192 18/30 Read block 277 len 32768 19/30 Write block 39 len 12288 data: 16 16 16 20/30 Write block 317 len 28672 data: 13 12 12 12 15 15 13 21/30 Write block 234 len 4096 data: 8 22/30 Read block 584 len 16384 23/30 Write block 342 len 36864 data: 17 17 20 20 19 17 20 19 14 24/30 Read block 360 len 8192 25/30 Read block 451 len 20480 26/30 Read block 187 len 12288 27/30 Read block 255 len 28672 28/30 Read block 84 len 24576 29/30 Write block 60 len 40960 data: 16 12 13 11 9 9 12 10 10 12 30/30 Write block 47 len 12288 data: 11 10 8 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Done false 1008 Acked Read 1 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Read 1 Done Done Done false 1011 Acked Write 3 Done Done Done false 1012 Acked Read 6 Done Done Done false 1013 Acked Write 7 Done Done Done false 1014 Acked Write 9 Done Done Done false 1015 Acked Write 6 Done Done Done false 1016 Acked Read 2 Done Done Done false 1017 Acked Read 8 Done Done Done false 1018 Acked Write 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Write 1 Done Done Done false 1021 Acked Read 4 Done Done Done false 1022 Acked Write 9 Done Done Done false 1023 Acked Read 2 Done Done Done false 1024 Acked Read 5 Done Done Done false 1025 Acked Read 3 Done Done Done false 1026 Acked Read 7 Done Done Done false 1027 Acked Read 6 Done Done Done false 1028 Acked Write 10 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 21 21 21 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1006 1006 1006 Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:01.419757306Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1926} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 182 182 182 310 310 310 T T T 2 040-059 182 182 182 310 310 310 T T T 3 060-079 178 178 178 304 304 304 T T T 4 080-099 178 178 178 302 302 302 F F F 5 100-119 177 177 177 298 298 298 T T T 6 120-139 181 181 181 308 308 308 F F F 7 140-159 186 186 186 315 315 315 F F F 8 160-179 186 186 186 315 315 315 T T T 9 180-199 186 186 186 315 315 315 F F F 10 200-219 184 184 184 312 312 312 T T T 11 220-239 182 182 182 310 310 310 T T T 12 240-259 182 182 182 310 310 310 F F F 13 260-279 183 183 183 311 311 311 T T T 14 280-299 183 183 183 311 311 311 F F F 15 300-319 175 175 175 296 296 296 T T T 16 320-339 183 183 183 311 311 311 T T T 17 340-359 182 182 182 310 310 310 T T T 18 360-379 186 186 186 315 315 315 F F F 19 380-399 186 186 186 315 315 315 F F F 20 400-419 184 184 184 313 313 313 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 186 186 186 315 315 315 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 174 174 174 295 295 295 T T T 29 580-599 185 185 185 314 314 314 T T T Max gen: 186, Max flush: 315 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 180 180 180 306 306 306 F F F 1 020-039 187 187 187 316 316 316 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 187 187 187 316 316 316 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 186 186 186 315 315 315 F F F 8 160-179 187 187 187 316 316 316 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 316 316 316 F F F 11 220-239 187 187 187 316 316 316 F F F 12 240-259 182 182 182 310 310 310 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 187 187 187 316 316 316 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 187 187 187 316 316 316 F F F 18 360-379 186 186 186 315 315 315 F F F 19 380-399 186 186 186 315 315 315 F F F 20 400-419 184 184 184 313 313 313 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 186 186 186 315 315 315 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 187, Max flush: 316 Begin loop 94 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.843260951Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.887671648Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.888345104Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.888398682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.888428841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.88845682Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} {"msg":"Crucible d94726fa-6702-42e3-aa2c-7f4f90afb12c has session id: 82daba42-3c0c-4505-9b2d-3278257c7c4e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.888904064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Upstairs opts: Upstairs UUID: d94726fa-6702-42e3-aa2c-7f4f90afb12c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.888937782Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Crucible stats registered with UUID: d94726fa-6702-42e3-aa2c-7f4f90afb12c","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.888953982Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919605015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919636294Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} {"msg":"d94726fa-6702-42e3-aa2c-7f4f90afb12c active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919697382Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919726191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.91976101Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919790549Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919820728Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919849437Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919873156Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919899825Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919929484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919956013Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:36169)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.919982822Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"io task","client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:41044)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920222423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"io task","client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46088)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920260162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"io task","client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920772863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920801202Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920816742Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920830941Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.920844681Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.92086324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.92141206Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.921579484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.921722999Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922006409Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922029038Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922043078Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922182093Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922203402Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922217191Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.92225516Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922272739Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[0]R flush_numbers[0..12]: [306, 316, 316, 316, 302, 316, 308, 315, 316, 315, 316, 316]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922293298Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[0]R generation[0..12]: [180, 187, 187, 187, 178, 187, 181, 186, 187, 186, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922307308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922321448Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[1]R flush_numbers[0..12]: [306, 316, 316, 316, 302, 316, 308, 315, 316, 315, 316, 316]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922335427Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[1]R generation[0..12]: [180, 187, 187, 187, 178, 187, 181, 186, 187, 186, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922349227Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922363576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[2]R flush_numbers[0..12]: [306, 316, 316, 316, 302, 316, 308, 315, 316, 315, 316, 316]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922380345Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[2]R generation[0..12]: [180, 187, 187, 187, 178, 187, 181, 186, 187, 186, 187, 187]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922394435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922408494Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922421734Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922435173Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922448433Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922461652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"":"downstairs","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Wait for a query_work_queue command to finish before sending IO No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922482262Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922494651Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"0","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922507751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"1","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922521Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"client":"2","":"downstairs","session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.92253445Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922546929Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"d94726fa-6702-42e3-aa2c-7f4f90afb12c is now active with session: 82daba42-3c0c-4505-9b2d-3278257c7c4e","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922559959Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922572119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934,"session_id":"82daba42-3c0c-4505-9b2d-3278257c7c4e"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:05.922584528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 259 len 4096 data: 11 02/30 Write block 31 len 20480 data: 20 16 15 18 14 03/30 Read block 218 len 40960 04/30 Read block 172 len 12288 05/30 Flush 06/30 Write block 150 len 12288 data: 13 12 15 07/30 Read block 525 len 4096 08/30 Read block 98 len 24576 09/30 Write block 79 len 4096 data: 10 10/30 Write block 19 len 28672 data: 13 15 13 16 11 13 18 11/30 Flush 12/30 Write block 160 len 32768 data: 13 11 14 14 14 16 17 15 13/30 Read block 521 len 36864 14/30 Write block 349 len 36864 data: 20 15 13 13 11 14 15 14 15 15/30 Read block 383 len 4096 16/30 Flush 17/30 Write block 389 len 16384 data: 12 9 11 14 18/30 Read block 62 len 24576 19/30 Write block 376 len 40960 data: 15 15 16 17 19 20 19 19 17 18 20/30 Read block 226 len 40960 21/30 Read block 459 len 24576 22/30 Write block 75 len 4096 data: 14 23/30 Write block 391 len 8192 data: 12 15 24/30 Read block 281 len 12288 25/30 Flush 26/30 Write block 346 len 32768 data: 20 18 21 21 16 14 14 12 27/30 Read block 233 len 8192 28/30 Read block 453 len 40960 29/30 Write block 239 len 36864 data: 13 13 10 11 12 16 14 15 13 30/30 Write block 243 len 16384 data: 13 17 15 16 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:5 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1025 Acked Write 8 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Read 10 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1024 1024 1024 Downstairs last five completed: 1024 1023 1022 1021 1020 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:05.950181532Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1934} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 188 188 188 318 318 318 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 188 188 188 320 320 320 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 188 188 188 318 318 318 F F F 8 160-179 188 188 188 319 319 319 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 316 316 316 F F F 11 220-239 187 187 187 316 316 316 T T T 12 240-259 188 188 188 317 317 317 T T T 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 187 187 187 316 316 316 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 188 188 188 319 319 319 T T T 18 360-379 188 188 188 320 320 320 F F F 19 380-399 188 188 188 320 320 320 F F F 20 400-419 184 184 184 313 313 313 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 186 186 186 315 315 315 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 188, Max flush: 320 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 188 188 188 318 318 318 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 188 188 188 320 320 320 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 188 188 188 318 318 318 F F F 8 160-179 188 188 188 319 319 319 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 316 316 316 F F F 11 220-239 189 189 189 321 321 321 F F F 12 240-259 189 189 189 321 321 321 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 187 187 187 316 316 316 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 188 188 188 320 320 320 F F F 19 380-399 188 188 188 320 320 320 F F F 20 400-419 184 184 184 313 313 313 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 186 186 186 315 315 315 F F F 23 460-479 180 180 180 307 307 307 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 189, Max flush: 321 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.241827058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.285741482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.286426928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.286480216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.286502945Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.286526884Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} {"msg":"Crucible 26c34179-540b-4bb3-82dd-96e139797098 has session id: 92da4935-cf9d-4316-a3e3-98c8a0e0e033","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.286982158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Upstairs opts: Upstairs UUID: 26c34179-540b-4bb3-82dd-96e139797098, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.287027406Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Crucible stats registered with UUID: 26c34179-540b-4bb3-82dd-96e139797098","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.287052515Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.31711184Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317145189Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} {"msg":"26c34179-540b-4bb3-82dd-96e139797098 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317226996Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317249965Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317277714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317295714Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317312543Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317329512Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317348241Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317380151Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317414799Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317446548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_connection connected from Ok(127.0.0.1:35606)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317483267Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"io task","client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_connection connected from Ok(127.0.0.1:56422)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317694719Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"io task","client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_connection connected from Ok(127.0.0.1:57066)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.317739238Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"io task","client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318119004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318148393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318297457Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318342286Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318373204Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318400383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318729392Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.318864487Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319140087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319305721Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.31932604Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.31934004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319478325Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319501664Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319515553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319657158Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319676597Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 316, 320, 302, 316, 308, 318, 319, 315, 316, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319691367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[0]R generation[0..12]: [188, 188, 187, 188, 178, 187, 181, 188, 188, 186, 187, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319705316Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319719816Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 316, 320, 302, 316, 308, 318, 319, 315, 316, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319733626Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[1]R generation[0..12]: [188, 188, 187, 188, 178, 187, 181, 188, 188, 186, 187, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319747175Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319761145Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 316, 320, 302, 316, 308, 318, 319, 315, 316, 321]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319775004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[2]R generation[0..12]: [188, 188, 187, 188, 178, 187, 181, 188, 188, 186, 187, 189]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319792413Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319810013Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319826422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319846271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Next flush: 322","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319862911Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"All extents match","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:10:10.31987917Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"":"downstairs","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319910159Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319941338Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"0","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.319972407Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"1","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.320007245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"client":"2","":"downstairs","session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.320039555Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.320057324Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"26c34179-540b-4bb3-82dd-96e139797098 is now active with session: 92da4935-cf9d-4316-a3e3-98c8a0e0e033","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.320073373Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.320088463Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942,"session_id":"92da4935-cf9d-4316-a3e3-98c8a0e0e033"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:10.320103962Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 222 len 28672 data: 11 10 9 9 11 13 10 02/30 Read block 446 len 24576 03/30 Read block 88 len 8192 04/30 Read block 537 len 36864 05/30 Read block 572 len 40960 06/30 Read block 332 len 36864 07/30 Read block 507 len 8192 08/30 Write block 472 len 12288 data: 11 12 16 09/30 Read block 24 len 4096 10/30 Write block 445 len 16384 data: 9 12 15 13 11/30 Read block 104 len 4096 12/30 Write block 367 len 36864 data: 14 12 15 16 16 16 15 15 15 13/30 Read block 354 len 8192 14/30 Write block 151 len 16384 data: 13 16 12 13 15/30 Read block 311 len 8192 16/30 Read block 567 len 24576 17/30 Read block 411 len 28672 18/30 Read block 442 len 8192 19/30 Write block 396 len 36864 data: 19 20 18 18 16 14 12 8 8 20/30 Flush 21/30 Write block 205 len 20480 data: 17 19 16 15 16 22/30 Read block 115 len 12288 23/30 Write block 451 len 20480 data: 16 17 20 18 15 24/30 Read block 489 len 28672 25/30 Read block 391 len 4096 26/30 Read block 461 len 8192 27/30 Read block 94 len 16384 28/30 Read block 426 len 16384 29/30 Read block 396 len 16384 30/30 Read block 372 len 8192 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 5 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Write 5 Done Done Done false 1023 Acked Read 7 Done Done Done false 1024 Acked Read 1 Done Done Done false 1025 Acked Read 2 Done Done Done false 1026 Acked Read 4 Done Done Done false 1027 Acked Read 4 Done Done Done false 1028 Acked Read 4 Done Done Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 1 2 Done 10 9 9 28 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:10.342248303Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1942} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 188 188 188 318 318 318 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 188 188 188 320 320 320 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 188 188 188 319 319 319 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 187 187 187 316 316 316 T T T 11 220-239 190 190 190 322 322 322 F F F 12 240-259 189 189 189 321 321 321 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 187 187 187 316 316 316 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 190 190 190 322 322 322 F F F 20 400-419 190 190 190 322 322 322 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 190 190 190 322 322 322 T T T 23 460-479 190 190 190 322 322 322 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 190, Max flush: 322 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 188 188 188 318 318 318 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 188 188 188 320 320 320 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 188 188 188 319 319 319 F F F 9 180-199 186 186 186 315 315 315 F F F 10 200-219 191 191 191 323 323 323 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 189 189 189 321 321 321 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 187 187 187 316 316 316 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 190 190 190 322 322 322 F F F 20 400-419 190 190 190 322 322 322 F F F 21 420-439 180 180 180 307 307 307 F F F 22 440-459 191 191 191 323 323 323 F F F 23 460-479 190 190 190 322 322 322 F F F 24 480-499 176 176 176 297 297 297 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 F F F 27 540-559 184 184 184 312 312 312 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 191, Max flush: 323 Begin loop 96 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.611319064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:14.653172383Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.653558589Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.684196483Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.684718844Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.684762913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.684780292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.684798591Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} {"msg":"Crucible 41e30e74-7963-463f-9f43-48fdd6104ad0 has session id: 4951a0ff-643c-49db-99d1-cd5087a6fab5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.685286084Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Upstairs opts: Upstairs UUID: 41e30e74-7963-463f-9f43-48fdd6104ad0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.685322422Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Crucible stats registered with UUID: 41e30e74-7963-463f-9f43-48fdd6104ad0","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.685343252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715541412Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.71557491Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} {"msg":"41e30e74-7963-463f-9f43-48fdd6104ad0 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715611419Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715628319Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715655358Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715674167Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715691176Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715707806Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715724295Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715740775Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715758224Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715775523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:53206)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715837281Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"io task","client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:61755)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715880689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"io task","client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_connection connected from Ok(127.0.0.1:42882)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.715920438Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"io task","client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.716711499Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.716749718Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.716779127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.716819985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.716851194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.716877863Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.717351886Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.717381815Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.717397685Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.717907516Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.717948095Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.717978374Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.71807908Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718108639Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718129088Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718142388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718155947Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 316, 320, 302, 316, 308, 322, 319, 315, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718170027Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[0]R generation[0..12]: [188, 188, 187, 188, 178, 187, 181, 190, 188, 186, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718184036Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718198216Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 316, 320, 302, 316, 308, 322, 319, 315, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718212065Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[1]R generation[0..12]: [188, 188, 187, 188, 178, 187, 181, 190, 188, 186, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718228255Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718255164Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 316, 320, 302, 316, 308, 322, 319, 315, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718284023Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[2]R generation[0..12]: [188, 188, 187, 188, 178, 187, 181, 190, 188, 186, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718304642Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2024-11-07T14:10:14.718329991Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.71835846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718386229Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Next flush: 324","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718406058Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718427067Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"":"downstairs","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718453986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718479436Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"0","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718495985Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"1","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718514004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"client":"2","":"downstairs","session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718538834Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718564333Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"41e30e74-7963-463f-9f43-48fdd6104ad0 is now active with session: 4951a0ff-643c-49db-99d1-cd5087a6fab5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718586322Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718599931Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950,"session_id":"4951a0ff-643c-49db-99d1-cd5087a6fab5"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:14.718620211Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 61 len 8192 data: 13 14 02/30 Read block 410 len 28672 03/30 Write block 544 len 16384 data: 12 12 18 20 04/30 Read block 138 len 4096 05/30 Read block 105 len 24576 06/30 Write block 432 len 32768 data: 16 16 15 13 11 11 11 8 07/30 Read block 459 len 32768 08/30 Read block 162 len 16384 09/30 Flush 10/30 Write block 446 len 24576 data: 13 16 14 15 17 17 11/30 Read block 207 len 20480 12/30 Write block 313 len 12288 data: 8 8 7 13/30 Read block 203 len 36864 14/30 Read block 163 len 16384 15/30 Read block 246 len 4096 16/30 Read block 554 len 32768 17/30 Write block 495 len 4096 data: 12 18/30 Read block 529 len 4096 19/30 Read block 593 len 4096 20/30 Flush 21/30 Write block 189 len 36864 data: 16 13 12 12 14 16 13 12 12 22/30 Write block 174 len 16384 data: 10 11 11 11 23/30 Write block 66 len 32768 data: 13 11 11 13 12 13 11 10 24/30 Write block 537 len 40960 data: 11 12 8 11 11 9 8 13 13 19 25/30 Read block 399 len 12288 26/30 Write block 433 len 36864 data: 17 16 14 12 12 12 9 9 9 27/30 Read block 287 len 4096 28/30 Write block 254 len 8192 data: 17 19 29/30 Write block 458 len 36864 data: 14 17 14 19 19 19 17 17 16 30/30 Write block 69 len 12288 data: 14 13 14 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:10 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1020 Acked Write 9 Done Done Done false 1021 Acked Write 4 Done Done Done false 1022 Acked Write 8 Done Done Done false 1023 Acked Write 10 Done Done Done false 1024 Acked Read 3 Done Done Done false 1025 Acked Write 9 Done Done Done false 1026 Acked Read 1 Done Done Done false 1027 Acked Write 2 Done Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 2 6 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1019 1019 1019 Downstairs last five completed: 1019 1018 1017 1016 1015 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:14.741754715Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1950} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 188 188 188 318 318 318 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 192 192 192 324 324 324 T T T 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 188 188 188 319 319 319 T T T 9 180-199 186 186 186 315 315 315 T T T 10 200-219 191 191 191 323 323 323 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 189 189 189 321 321 321 T T T 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 190 190 190 322 322 322 F F F 20 400-419 190 190 190 322 322 322 F F F 21 420-439 192 192 192 324 324 324 T T T 22 440-459 192 192 192 325 325 325 T T T 23 460-479 190 190 190 322 322 322 T T T 24 480-499 192 192 192 325 325 325 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 184 184 184 313 313 313 T T T 27 540-559 192 192 192 324 324 324 T T T 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 192, Max flush: 325 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 188 188 188 318 318 318 F F F 2 040-059 187 187 187 316 316 316 F F F 3 060-079 193 193 193 326 326 326 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 F F F 7 140-159 190 190 190 322 322 322 F F F 8 160-179 193 193 193 326 326 326 F F F 9 180-199 193 193 193 326 326 326 F F F 10 200-219 191 191 191 323 323 323 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 190 190 190 322 322 322 F F F 20 400-419 190 190 190 322 322 322 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 193 193 193 326 326 326 F F F 23 460-479 193 193 193 326 326 326 F F F 24 480-499 192 192 192 325 325 325 F F F 25 500-519 181 181 181 308 308 308 F F F 26 520-539 193 193 193 326 326 326 F F F 27 540-559 193 193 193 326 326 326 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 F F F Max gen: 193, Max flush: 326 Begin loop 97 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.138249017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:19.180596048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.181006973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.211886519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.212481297Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.212510576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.212523846Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.212539245Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} {"msg":"Crucible aa6e449f-e4fb-4c0e-bd84-32f2128d9ce5 has session id: 19845203-c38d-49d9-99eb-9a2957acd4cc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.213011628Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Upstairs opts: Upstairs UUID: aa6e449f-e4fb-4c0e-bd84-32f2128d9ce5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.213062026Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Crucible stats registered with UUID: aa6e449f-e4fb-4c0e-bd84-32f2128d9ce5","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.213096185Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.242840021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.2428751Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} {"msg":"aa6e449f-e4fb-4c0e-bd84-32f2128d9ce5 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.242912199Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.242929258Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.242955877Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.242974127Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.242991446Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243072393Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243108972Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243142021Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243175389Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243211318Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59249)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243324194Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"io task","client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:59750)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.24343017Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"io task","client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:60163)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.243450139Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"io task","client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.24398119Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244017529Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244035938Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244050368Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244063957Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244077277Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244590408Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244730633Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.244894457Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245145118Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245179787Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245206566Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245311302Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245343771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.24536992Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245402749Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245418208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 316, 326, 302, 316, 308, 322, 326, 326, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245457767Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[0]R generation[0..12]: [188, 188, 187, 193, 178, 187, 181, 190, 193, 193, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245476236Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245490636Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 316, 326, 302, 316, 308, 322, 326, 326, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245504625Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[1]R generation[0..12]: [188, 188, 187, 193, 178, 187, 181, 190, 193, 193, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245518435Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245532304Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 316, 326, 302, 316, 308, 322, 326, 326, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245546004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[2]R generation[0..12]: [188, 188, 187, 193, 178, 187, 181, 190, 193, 193, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245559723Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245573623Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245586902Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245600942Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Next flush: 327","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245614231Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245629531ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"":"downstairs","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.2456523Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245664639Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"0","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245678279Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"1","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245691649Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"client":"2","":"downstairs","session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245705048Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245717588Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"aa6e449f-e4fb-4c0e-bd84-32f2128d9ce5 is now active with session: 19845203-c38d-49d9-99eb-9a2957acd4cc","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245730477Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245742687Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958,"session_id":"19845203-c38d-49d9-99eb-9a2957acd4cc"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:19.245754986Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 534 len 32768 data: 13 12 11 12 13 9 12 12 02/30 Read block 19 len 4096 03/30 Write block 398 len 4096 data: 19 04/30 Read block 474 len 4096 05/30 Write block 506 len 24576 data: 14 11 13 14 18 19 06/30 Flush 07/30 Write block 22 len 16384 data: 17 12 14 19 08/30 Write block 465 len 16384 data: 18 17 16 14 09/30 Flush 10/30 Write block 446 len 32768 data: 14 17 15 16 18 18 18 21 11/30 Write block 42 len 24576 data: 13 13 13 14 13 12 12/30 Read block 11 len 20480 13/30 Read block 130 len 8192 14/30 Read block 447 len 36864 15/30 Read block 6 len 28672 16/30 Write block 124 len 12288 data: 18 20 22 17/30 Write block 174 len 40960 data: 11 12 12 12 11 14 14 14 12 13 18/30 Read block 61 len 16384 19/30 Read block 542 len 20480 20/30 Read block 178 len 16384 21/30 Read block 29 len 8192 22/30 Write block 581 len 12288 data: 10 11 15 23/30 Read block 143 len 24576 24/30 Write block 477 len 24576 data: 8 5 7 8 11 11 25/30 Read block 63 len 24576 26/30 Write block 140 len 4096 data: 12 27/30 Read block 208 len 12288 28/30 Write block 510 len 12288 data: 19 20 18 29/30 Read block 411 len 32768 30/30 Write block 490 len 12288 data: 11 12 13 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 8 Done Done Done false 1010 Acked Write 6 Done Done Done false 1011 Acked Read 5 Done Done Done false 1012 Acked Read 2 Done Done Done false 1013 Acked Read 9 Done Done Done false 1014 Acked Read 7 Done Done Done false 1015 Acked Write 3 Done Done Done false 1016 Acked Write 10 Done Done Done false 1017 Acked Read 4 Done Done Done false 1018 Acked Read 5 Done Done Done false 1019 Acked Read 4 Done Done Done false 1020 Acked Read 2 Done Done Done false 1021 Acked Write 3 Done Done Done false 1022 Acked Read 6 Done Done Done false 1023 Acked Write 6 Done Done Done false 1024 Acked Read 6 Done Done Done false 1025 Acked Write 1 Done Done Done false 1026 Acked Read 3 Done Done Done false 1027 Acked Write 3 Done Done Done false 1028 Acked Read 8 Done Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 20 20 20 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1008 1008 1008 Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:19.269142092Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1958} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 194 194 194 328 328 328 F F F 2 040-059 187 187 187 316 316 316 T T T 3 060-079 193 193 193 326 326 326 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 181 181 181 308 308 308 T T T 7 140-159 190 190 190 322 322 322 T T T 8 160-179 193 193 193 326 326 326 T T T 9 180-199 193 193 193 326 326 326 T T T 10 200-219 191 191 191 323 323 323 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 190 190 190 322 322 322 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 193 193 193 326 326 326 T T T 23 460-479 194 194 194 328 328 328 T T T 24 480-499 192 192 192 325 325 325 T T T 25 500-519 194 194 194 327 327 327 T T T 26 520-539 194 194 194 327 327 327 F F F 27 540-559 194 194 194 327 327 327 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 187 187 187 316 316 316 T T T Max gen: 194, Max flush: 328 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 188 188 188 318 318 318 F F F 1 020-039 194 194 194 328 328 328 F F F 2 040-059 195 195 195 329 329 329 F F F 3 060-079 193 193 193 326 326 326 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 195 195 195 329 329 329 F F F 7 140-159 195 195 195 329 329 329 F F F 8 160-179 195 195 195 329 329 329 F F F 9 180-199 195 195 195 329 329 329 F F F 10 200-219 191 191 191 323 323 323 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 193 193 193 326 326 326 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 192 192 192 325 325 325 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 189 189 189 321 321 321 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 190 190 190 322 322 322 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 195 195 195 329 329 329 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 195 195 195 329 329 329 F F F 26 520-539 194 194 194 327 327 327 F F F 27 540-559 194 194 194 327 327 327 F F F 28 560-579 187 187 187 316 316 316 F F F 29 580-599 195 195 195 329 329 329 F F F Max gen: 195, Max flush: 329 Begin loop 98 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.681935405Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:23.725683565Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.726082041Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.756878859Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.757521096Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.757557485Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.757572884Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.757587404Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} {"msg":"Crucible 20dfdc83-65d4-413b-8f52-42f36488b43f has session id: 410fd6aa-4a53-418b-a6f9-f51077109ba2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.758034607Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Upstairs opts: Upstairs UUID: 20dfdc83-65d4-413b-8f52-42f36488b43f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.758081576Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Crucible stats registered with UUID: 20dfdc83-65d4-413b-8f52-42f36488b43f","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.758107615Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787632818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787671727Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} {"msg":"20dfdc83-65d4-413b-8f52-42f36488b43f active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787704656Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787726005Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787771453Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787806362Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787841091Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787877349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787911168Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787945807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.787981626Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788018154Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51824)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788071372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"io task","client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63642)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788171689Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"io task","client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_connection connected from Ok(127.0.0.1:42252)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788196558Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"io task","client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788794756Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788828735Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788847144Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788863934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788880603Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.788897283Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.789481822Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.78951137Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.78952811Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790110909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790143528Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790159367Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790172377Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790189246Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790202956Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790215045Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790228284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[0]R flush_numbers[0..12]: [318, 328, 329, 326, 302, 316, 329, 329, 329, 329, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790242014Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[0]R generation[0..12]: [188, 194, 195, 193, 178, 187, 195, 195, 195, 195, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790255934Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790269973Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[1]R flush_numbers[0..12]: [318, 328, 329, 326, 302, 316, 329, 329, 329, 329, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790283673Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[1]R generation[0..12]: [188, 194, 195, 193, 178, 187, 195, 195, 195, 195, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790297292Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790311321Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[2]R flush_numbers[0..12]: [318, 328, 329, 326, 302, 316, 329, 329, 329, 329, 323, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790325001Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[2]R generation[0..12]: [188, 194, 195, 193, 178, 187, 195, 195, 195, 195, 191, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790338611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.79035252Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.79036553Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790380449Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":Wait for a query_work_queue command to finish before sending IO 1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Next flush: 330","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790412208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790441087Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"":"downstairs","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790469586Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790496015Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"0","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790525484Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"1","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790554913Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"client":"2","":"downstairs","session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790585002Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790612161Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66"Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"20dfdc83-65d4-413b-8f52-42f36488b43f is now active with session: 410fd6aa-4a53-418b-a6f9-f51077109ba2","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790644909Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790665519Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966,"session_id":"410fd6aa-4a53-418b-a6f9-f51077109ba2"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:23.790678548Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 348 len 36864 data: 22 22 17 15 15 13 15 16 15 02/30 Write block 358 len 4096 data: 15 03/30 Write block 578 len 24576 data: 11 10 11 11 12 16 04/30 Read block 440 len 24576 05/30 Flush 06/30 Write block 66 len 16384 data: 14 12 12 15 07/30 Read block 123 len 40960 08/30 Read block 292 len 40960 09/30 Read block 93 len 40960 10/30 Read block 473 len 28672 11/30 Read block 238 len 40960 12/30 Write block 13 len 32768 data: 11 11 9 11 13 12 14 16 13/30 Flush 14/30 Write block 201 len 8192 data: 10 13 15/30 Write block 302 len 20480 data: 10 12 10 11 8 16/30 Write block 409 len 28672 data: 16 17 16 16 16 13 11 17/30 Write block 544 len 28672 data: 14 14 20 21 24 22 19 18/30 Read block 431 len 8192 19/30 Read block 521 len 12288 20/30 Write block 241 len 28672 data: 11 12 14 18 16 17 14 21/30 Read block 6 len 36864 22/30 Read block 110 len 12288 23/30 Read block 366 len 36864 24/30 Read block 545 len 12288 25/30 Read block 355 len 40960 26/30 Read block 110 len 16384 27/30 Read block 104 len 8192 28/30 Write block 23 len 32768 data: 13 15 20 17 15 16 18 17 29/30 Read block 220 len 36864 30/30 Read block 61 len 8192 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:17 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1013 Acked Write 2 Done Done Done false 1014 Acked Write 5 Done Done Done false 1015 Acked Write 7 Done Done Done false 1016 Acked Write 7 Done Done Done false 1017 Acked Read 2 Done Done Done false 1018 Acked Read 3 Done Done Done false 1019 Acked Write 7 Done Done Done false 1020 Acked Read 9 Done Done Done false 1021 Acked Read 3 Done Done Done false 1022 Acked Read 9 Done Done Done false 1023 Acked Read 3 Done Done Done false 1024 Acked Read 10 Done Done Done false 1025 Acked Read 4 Done Done Done false 1026 Acked Read 2 Done Done Done false 1027 Acked Write 8 Done Done Done false 1028 Acked Read 9 Done Done Done false 1029 Acked Read 2 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 17 17 17 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1012 1012 1012 Downstairs last five completed: 1012 1011 1010 1009 1008 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:23.81417253Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1966} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 331 331 331 F F F 1 020-039 196 196 196 331 331 331 T T T 2 040-059 195 195 195 329 329 329 F F F 3 060-079 196 196 196 331 331 331 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 195 195 195 329 329 329 F F F 7 140-159 195 195 195 329 329 329 F F F 8 160-179 195 195 195 329 329 329 F F F 9 180-199 195 195 195 329 329 329 F F F 10 200-219 191 191 191 323 323 323 T T T 11 220-239 190 190 190 322 322 322 F F F 12 240-259 193 193 193 326 326 326 T T T 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 192 192 192 325 325 325 T T T 16 320-339 187 187 187 316 316 316 F F F 17 340-359 196 196 196 330 330 330 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 190 190 190 322 322 322 T T T 21 420-439 193 193 193 326 326 326 F F F 22 440-459 195 195 195 329 329 329 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 195 195 195 329 329 329 F F F 26 520-539 194 194 194 327 327 327 F F F 27 540-559 194 194 194 327 327 327 T T T 28 560-579 196 196 196 330 330 330 F F F 29 580-599 196 196 196 330 330 330 F F F Max gen: 196, Max flush: 331 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 331 331 331 F F F 1 020-039 197 197 197 332 332 332 F F F 2 040-059 195 195 195 329 329 329 F F F 3 060-079 196 196 196 331 331 331 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 195 195 195 329 329 329 F F F 7 140-159 195 195 195 329 329 329 F F F 8 160-179 195 195 195 329 329 329 F F F 9 180-199 195 195 195 329 329 329 F F F 10 200-219 197 197 197 332 332 332 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 197 197 197 332 332 332 F F F 13 260-279 187 187 187 316 316 316 F F F 14 280-299 183 183 183 311 311 311 F F F 15 300-319 197 197 197 332 332 332 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 196 196 196 330 330 330 F F F 18 360-379 190 190 190 322 322 322 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 197 197 197 332 332 332 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 195 195 195 329 329 329 F F F 23 460-479 195 195 195 329 329 329 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 195 195 195 329 329 329 F F F 26 520-539 194 194 194 327 327 327 F F F 27 540-559 197 197 197 332 332 332 F F F 28 560-579 196 196 196 330 330 330 F F F 29 580-599 196 196 196 330 330 330 F F F Max gen: 197, Max flush: 332 Begin loop 99 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.168275952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.211064067Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.211498342Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.242080818Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.242715705Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"311c3ca2c97d568407ae06780269d96fcfdd3a7c\",\n git_commit_timestamp: \"2024-11-07T13:47:04.000000000Z\",\n git_branch: \"renovate/tokio-1.x-lockfile\",\n rustc_semver: \"1.82.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"f6e511eec7342f59a25f7c0534f1dbea00d01b14\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.242750804Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} {"msg":"Upstairs <-> Downstairs Message Version: 12","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.242766073Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.242780423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} {"msg":"Crucible 43b9d28f-b31f-4713-a7d6-ecca7b24a037 has session id: 61037c99-e8c5-43c4-876a-7f549e46671b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.243287004Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Upstairs opts: Upstairs UUID: 43b9d28f-b31f-4713-a7d6-ecca7b24a037, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.243332352Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Crucible stats registered with UUID: 43b9d28f-b31f-4713-a7d6-ecca7b24a037","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.243354952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.272942064Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.272989162Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} {"msg":"43b9d28f-b31f-4713-a7d6-ecca7b24a037 active request set","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273061289Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273108388Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273160376Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273198854Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273234193Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273255172Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"client set_active_request while in New; waiting...","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273272842Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273293771Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.27332611Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273352759Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_connection connected from Ok(127.0.0.1:37117)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273378128Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"io task","client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_connection connected from Ok(127.0.0.1:58059)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273500534Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"io task","client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_connection connected from Ok(127.0.0.1:65007)","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.273545222Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"io task","client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274112952Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[1] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.27415118Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274185349Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[2] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274223308Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274260526Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[0] Transition from New to WaitActive","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274295085Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.27471372Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274795247Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.274827186Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.27526777Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[1] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275296979Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275313208Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275338857Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[2] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275357596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275371296Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"ds_transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275433284Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[0] Transition from WaitActive to WaitQuorum","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275452423Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[0]R flush_numbers[0..12]: [331, 332, 329, 331, 302, 316, 329, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275466612Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[0]R generation[0..12]: [196, 197, 195, 196, 178, 187, 195, 195, 195, 195, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275480642Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275494522Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[1]R flush_numbers[0..12]: [331, 332, 329, 331, 302, 316, 329, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275508271Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[1]R generation[0..12]: [196, 197, 195, 196, 178, 187, 195, 195, 195, 195, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275521841Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.27553567Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[2]R flush_numbers[0..12]: [331, 332, 329, 331, 302, 316, 329, 329, 329, 329, 332, 322]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275549369Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[2]R generation[0..12]: [196, 197, 195, 196, 178, 187, 195, 195, 195, 195, 197, 190]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275563039Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275576928Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275589918Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275605807Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Next flush: 333","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275619177Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275632337Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"":"downstairs","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275645536Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275657596Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"0","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":Wait for a query_work_queue command to finish before sending IO 30,"time":"2024-11-07T14:10:28.275694774Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"1","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275708754Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"client":"2","":"downstairs","session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275722203Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275734693Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"43b9d28f-b31f-4713-a7d6-ecca7b24a037 is now active with session: 61037c99-e8c5-43c4-876a-7f549e46671b","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275747482Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275759462Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974,"session_id":"61037c99-e8c5-43c4-876a-7f549e46671b"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2024-11-07T14:10:28.275771652Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} Region: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Loading write count information from file "/tmp/test_repair_verify.data" Run Repair workload 01/30 Write block 247 len 24576 data: 15 15 17 16 18 19 02/30 Write block 296 len 20480 data: 15 15 16 14 10 03/30 Read block 589 len 24576 04/30 Write block 47 len 12288 data: 13 11 9 05/30 Write block 349 len 4096 data: 23 06/30 Read block 37 len 12288 07/30 Read block 105 len 36864 08/30 Write block 458 len 36864 data: 15 18 15 20 20 20 18 19 18 09/30 Read block 460 len 16384 10/30 Flush 11/30 Write block 587 len 16384 data: 16 19 21 22 12/30 Read block 62 len 12288 13/30 Read block 487 len 4096 14/30 Read block 315 len 16384 15/30 Write block 58 len 16384 data: 15 14 17 14 16/30 Write block 120 len 40960 data: 18 17 16 16 19 21 23 23 23 18 17/30 Read block 218 len 12288 18/30 Flush 19/30 Write block 13 len 12288 data: 12 12 10 20/30 Write block 410 len 20480 data: 18 17 17 17 14 21/30 Flush 22/30 Write block 268 len 20480 data: 13 13 15 17 18 23/30 Read block 357 len 28672 24/30 Read block 180 len 20480 25/30 Write block 54 len 40960 data: 12 10 13 13 16 15 18 15 15 12 26/30 Read block 298 len 28672 27/30 Read block 86 len 24576 28/30 Write block 508 len 4096 data: 14 29/30 Read block 242 len 20480 30/30 Write block 369 len 4096 data: 16 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:9 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1021 Acked Write 5 Done Done Done false 1022 Acked Read 7 Done Done Done false 1023 Acked Read 5 Done Done Done false 1024 Acked Write 10 Done Done Done false 1025 Acked Read 7 Done Done Done false 1026 Acked Read 6 Done Done Done false 1027 Acked Write 1 Done Done Done false 1028 Acked Read 5 Done Done Done false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 1 3 Done 8 8 8 24 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: 1020 1020 1020 Downstairs last five completed: 1020 1019 1018 1017 1016 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2024-11-07T14:10:28.301278081Z","hostname":"w-01JC3EWRZ50H0J627E1M8D0Q66","pid":1974} Wrote out file "/tmp/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 335 335 335 F F F 1 020-039 197 197 197 332 332 332 F F F 2 040-059 198 198 198 334 334 334 T T T 3 060-079 198 198 198 334 334 334 T T T 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 198 198 198 334 334 334 F F F 7 140-159 195 195 195 329 329 329 F F F 8 160-179 195 195 195 329 329 329 F F F 9 180-199 195 195 195 329 329 329 F F F 10 200-219 197 197 197 332 332 332 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 198 198 198 333 333 333 F F F 13 260-279 187 187 187 316 316 316 T T T 14 280-299 198 198 198 333 333 333 F F F 15 300-319 198 198 198 333 333 333 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 198 198 198 333 333 333 F F F 18 360-379 190 190 190 322 322 322 T T T 19 380-399 194 194 194 327 327 327 F F F 20 400-419 198 198 198 335 335 335 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 198 198 198 333 333 333 F F F 23 460-479 198 198 198 333 333 333 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 195 195 195 329 329 329 T T T 26 520-539 194 194 194 327 327 327 F F F 27 540-559 197 197 197 332 332 332 F F F 28 560-579 196 196 196 330 330 330 F F F 29 580-599 198 198 198 334 334 334 F F F Max gen: 198, Max flush: 335 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 335 335 335 F F F 1 020-039 197 197 197 332 332 332 F F F 2 040-059 199 199 199 336 336 336 F F F 3 060-079 199 199 199 336 336 336 F F F 4 080-099 178 178 178 302 302 302 F F F 5 100-119 187 187 187 316 316 316 F F F 6 120-139 198 198 198 334 334 334 F F F 7 140-159 195 195 195 329 329 329 F F F 8 160-179 195 195 195 329 329 329 F F F 9 180-199 195 195 195 329 329 329 F F F 10 200-219 197 197 197 332 332 332 F F F 11 220-239 190 190 190 322 322 322 F F F 12 240-259 198 198 198 333 333 333 F F F 13 260-279 199 199 199 336 336 336 F F F 14 280-299 198 198 198 333 333 333 F F F 15 300-319 198 198 198 333 333 333 F F F 16 320-339 187 187 187 316 316 316 F F F 17 340-359 198 198 198 333 333 333 F F F 18 360-379 199 199 199 336 336 336 F F F 19 380-399 194 194 194 327 327 327 F F F 20 400-419 198 198 198 335 335 335 F F F 21 420-439 193 193 193 326 326 326 F F F 22 440-459 198 198 198 333 333 333 F F F 23 460-479 198 198 198 333 333 333 F F F 24 480-499 195 195 195 329 329 329 F F F 25 500-519 199 199 199 336 336 336 F F F 26 520-539 194 194 194 327 327 327 F F F 27 540-559 197 197 197 332 332 332 F F F 28 560-579 196 196 196 330 330 330 F F F 29 580-599 198 198 198 334 334 334 F F F Max gen: 199, Max flush: 336 7:30 Test duration Test completed real 7:30.453719870 user 56.523159394 sys 19.285250785 trap 0.069452146 tflt 0.055006979 dflt 0.088297214 kflt 0.005551290 lock 11:52:57.367925938 slp 36:26.737548267 lat 5.793969376 stop 0.059222960