Turn off color for downstairs dump Removing existing region "/var/tmp/test_repair" Creating region directory at: /var/tmp/test_repair Creating dsc directory at: /tmp/test_live_repair/test_repair_dsc Downstairs region 0 created at /var/tmp/test_repair/8810 in 0.24613917 Downstairs region 1 created at /var/tmp/test_repair/8820 in 0.015837006 Downstairs region 2 created at /var/tmp/test_repair/8830 in 0.014772708 Created 3 regions /var/tmp/bins/crutest with -t 127.0.0.1:8810 -t 127.0.0.1:8820 -t 127.0.0.1:8830 1201 1202 1203 Dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.476510899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.494629536Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.496423329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.49658167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.496595874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.496613986Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} {"msg":"Crucible 1b37f4b9-bf1a-4cac-af6c-a01fcd64fc4b has session id: 74cf3a27-a1b8-45ac-9e98-a773b01ab605","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.497155921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Upstairs opts: Upstairs UUID: 1b37f4b9-bf1a-4cac-af6c-a01fcd64fc4b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.497299089Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Crucible stats registered with UUID: 1b37f4b9-bf1a-4cac-af6c-a01fcd64fc4b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.497316062Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:48:56.497444976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498227737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.49826594Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498284282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498466943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"task":"crutest"} {"msg":"The guest has requested activation with gen:1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498653472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} {"msg":"1b37f4b9-bf1a-4cac-af6c-a01fcd64fc4b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498708299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498722053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"0","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498736516Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"1","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.49875044Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"2","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498791812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.498807596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.4988215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"ds_connection connected from Ok(127.0.0.1:53012)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.499150737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"0","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"ds_connection connected from Ok(127.0.0.1:62975)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.499178465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"2","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"ds_connection connected from Ok(127.0.0.1:45286)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.499194508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"io task","client":"1","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.501433478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"1","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.50149721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"2","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.501517651Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"0","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502159013Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502214189Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[0]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.50228241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[0]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502299012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502315665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[1]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502330649Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[1]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502345662Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502361915Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[2]R flush_numbers[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502376809Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[2]R generation[0..12]: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502391682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502407795Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Max found gen is 1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.50242098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Generation requested: 1 >= found:1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502434933Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Next flush: 1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502448128Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502471997Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"":"downstairs","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502486261Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502498766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"0","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.50251293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"1","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502526714Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"client":"2","":"downstairs","session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502540948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502554232Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"1b37f4b9-bf1a-4cac-af6c-a01fcd64fc4b is now active with session: 74cf3a27-a1b8-45ac-9e98-a773b01ab605","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502569115Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"The guest has finished waiting for activation with:1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.50258245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502593005Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206,"session_id":"74cf3a27-a1b8-45ac-9e98-a773b01ab605"} {"msg":"Activated sub_volume 1b37f4b9-bf1a-4cac-af6c-a01fcd64fc4b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:48:56.502606059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} Wait for a query_work_queue command to finish before sending IO Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Fill test Read and Verify all blocks (0..600 range:false) Wrote out file "/tmp/test_live_repair/test_repair_verify.data" CLIENT: Tests done. All submitted work has been ACK'd ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:3 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Read 256 Done Done Done false 1005 Acked Read 256 Done Done Done false 1006 Acked Read 88 Done Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 0 0 Done 3 3 3 9 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1006 1005 1004 1003 1002 CLIENT: Up:0 ds:3 act:3 ---------------------------------------------------------------- Crucible gen:1 GIO:true work queues: Upstairs:0 downstairs:0 Downstairs last five completed: 1007 1006 1005 1004 1003 Upstairs last five completed: 1007 1006 1005 1004 1003 CLIENT: Up:0 ds:0 act:3 CLIENT: All crucible jobs finished, exiting program {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:00.574308134Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1206} Begin loop 1 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.600428304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:00.611134984Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.611195068Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.621813226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622101031Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622142064Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622153948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622171671Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} {"msg":"Crucible f92f1496-27c0-4f08-94a0-306bc02a812d has session id: 2e9cabb2-4300-4df4-b5c4-56b3303a53ba","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622514412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Upstairs opts: Upstairs UUID: f92f1496-27c0-4f08-94a0-306bc02a812d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622550097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Crucible stats registered with UUID: f92f1496-27c0-4f08-94a0-306bc02a812d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.62256609Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:00.622579624Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622756977Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622785345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622800348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622917827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"task":"crutest"} {"msg":"The guest has requested activation with gen:2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.62293471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} {"msg":"f92f1496-27c0-4f08-94a0-306bc02a812d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622951483Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622964517Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"0","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622978361Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"1","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.622992825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"2","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623017144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623031528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623045832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:55653)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623092941Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"2","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:49566)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623121049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"0","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"ds_connection connected from Ok(127.0.0.1:48011)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623189789Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"io task","client":"1","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623833261Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"2","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623861418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"1","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.623886018Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"0","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624214425Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624263424Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[0]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624317301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[0]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624333914Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624350886Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[1]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.62436618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[1]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624381323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624397726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[2]R flush_numbers[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624412699Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[2]R generation[0..12]: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624427813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624444106Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Max found gen is 2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.62445777Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Generation requested: 2 >= found:2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624471734Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Next flush: 2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624493175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624506919Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"":"downstairs","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624520823Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624533577Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"0","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624547581Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"1","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624561555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"client":"2","":"downstairs","session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624575439Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624591882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"f92f1496-27c0-4f08-94a0-306bc02a812d is now active with session: 2e9cabb2-4300-4df4-b5c4-56b3303a53ba","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624606895Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.62461968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208,"session_id":"2e9cabb2-4300-4df4-b5c4-56b3303a53ba"} {"msg":"The guest has finished waiting for activation with:2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624632884Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} {"msg":"Activated sub_volume f92f1496-27c0-4f08-94a0-306bc02a812d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:00.624644359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 553 len 4096 data: 2 02/30 Write block 253 len 40960 data: 2 2 2 2 2 2 2 2 2 2 03/30 Read block 49 len 40960 04/30 Flush 05/30 Write block 296 len 8192 data: 2 2 06/30 Write block 330 len 36864 data: 2 2 2 2 2 2 2 2 2 07/30 Write block 93 len 16384 data: 2 2 2 2 08/30 Read block 188 len 32768 09/30 Write block 398 len 4096 data: 2 10/30 Write block 402 len 32768 data: 2 2 2 2 2 2 2 2 11/30 Write block 517 len 36864 data: 2 2 2 2 2 2 2 2 2 12/30 Write block 381 len 8192 data: 2 2 13/30 Read block 5 len 8192 14/30 Read block 227 len 8192 15/30 Write block 384 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Write block 307 len 40960 data: 2 2 2 2 2 2 2 2 2 2 17/30 Read block 521 len 28672 18/30 Write block 437 len 12288 data: 2 2 2 19/30 Read block 290 len 24576 20/30 Read block 397 len 28672 21/30 Read block 584 len 40960 22/30 Flush 23/30 Flush 24/30 Write block 495 len 24576 data: 2 2 2 2 2 2 25/30 Read block 27 len 36864 26/30 Write block 30 len 28672 data: 2 2 2 2 2 2 2 27/30 Read block 131 len 4096 28/30 Read block 559 len 40960 29/30 Read block 12 len 8192 30/30 Write block 385 len 36864 data: 3 3 3 3 3 3 3 3 3 ---------------------------------------------------------------- Crucible gen:2 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Write 9 Done Done Done false 1006 Acked Write 4 Done Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 7 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 6 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Read 1 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 3 25 25 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:00.63721784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1208} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 1 1 1 1 1 1 F T T <--- 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 1 2 2 1 3 3 T F F <--- 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 1 2 2 1 3 3 T F F <--- 15 300-319 1 2 2 1 3 3 F F F <--- 16 320-339 1 2 2 1 3 3 T F F <--- 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 1 2 2 1 3 3 F T T <--- 20 400-419 1 2 2 1 3 3 F F F <--- 21 420-439 1 2 2 1 3 3 F F F <--- 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 1 1 1 1 1 1 F T T <--- 25 500-519 1 2 2 1 3 3 F T T <--- 26 520-539 1 2 2 1 3 3 F F F <--- 27 540-559 2 2 2 2 2 2 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 2, Max flush: 3 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 1 Verifying data now No change after verify Loop: 1 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 1 1 1 1 1 1 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 F F F 7 140-159 1 1 1 1 1 1 F F F 8 160-179 1 1 1 1 1 1 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 3 3 3 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 2 2 2 3 3 3 F F F 22 440-459 1 1 1 1 1 1 F F F 23 460-479 1 1 1 1 1 1 F F F 24 480-499 3 3 3 4 4 4 F F F 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 2 2 2 F F F 28 560-579 1 1 1 1 1 1 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Begin loop 2 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.118623222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.13055654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.130885117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.1309247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.130936815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.130954657Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} {"msg":"Crucible 298bb5bf-5f5e-4f8b-8eef-1b3e800fac6b has session id: 46813cb3-88ab-49df-b16e-fe4e12d9d2c3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131333903Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Upstairs opts: Upstairs UUID: 298bb5bf-5f5e-4f8b-8eef-1b3e800fac6b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131368608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Crucible stats registered with UUID: 298bb5bf-5f5e-4f8b-8eef-1b3e800fac6b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131384161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:05.131397665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131563373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131589222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131603986Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131740677Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"task":"crutest"} {"msg":"The guest has requested activation with gen:4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131757419Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} {"msg":"298bb5bf-5f5e-4f8b-8eef-1b3e800fac6b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131774532Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131787096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"0","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.13180133Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"1","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131815134Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"2","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131828788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131843052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131859865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57375)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.13189428Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"2","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:57007)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.131910843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"0","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"ds_connection connected from Ok(127.0.0.1:47416)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.13205657Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"io task","client":"1","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.132659059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"2","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.132700301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"0","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.132718453Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"1","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133045451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133061464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[0]R flush_numbers[0..12]: [1, 4, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133081495Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[0]R generation[0..12]: [1, 3, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133096449Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133112982Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[1]R flush_numbers[0..12]: [1, 4, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133128155Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[1]R generation[0..12]: [1, 3, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133143199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.13316302Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[2]R flush_numbers[0..12]: [1, 4, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133178174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[2]R generation[0..12]: [1, 3, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133193067Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.13320917Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Max found gen is 4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133222494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Generation requested: 4 >= found:4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133236608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Next flush: 5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133250162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133263637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"":"downstairs","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:49:05.13327802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133294563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"0","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133308697Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"1","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133322601Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"client":"2","":"downstairs","session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133336815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133350199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"298bb5bf-5f5e-4f8b-8eef-1b3e800fac6b is now active with session: 46813cb3-88ab-49df-b16e-fe4e12d9d2c3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133364923Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"Set Active after no reconciliationDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133382595Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216,"session_id":"46813cb3-88ab-49df-b16e-fe4e12d9d2c3"} {"msg":"The guest has finished waiting for activation with:4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133395999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} {"msg":"Activated sub_volume 298bb5bf-5f5e-4f8b-8eef-1b3e800fac6b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:05.133407434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 565 len 20480 data: 2 2 2 2 2 02/30 Write block 158 len 24576 data: 2 2 2 2 2 2 03/30 Read block 251 len 28672 04/30 Read block 116 len 36864 05/30 Write block 145 len 28672 data: 2 2 2 2 2 2 2 06/30 Read block 228 len 28672 07/30 Read block 584 len 32768 08/30 Read block 293 len 4096 09/30 Read block 168 len 32768 10/30 Read block 359 len 16384 11/30 Write block 461 len 4096 data: 2 12/30 Read block 282 len 8192 13/30 Read block 496 len 36864 14/30 Read block 3 len 12288 15/30 Write block 228 len 40960 data: 2 2 2 2 2 2 2 2 2 2 16/30 Read block 252 len 32768 17/30 Write block 547 len 40960 data: 2 2 2 2 2 2 3 2 2 2 18/30 Read block 299 len 24576 19/30 Read block 56 len 36864 20/30 Write block 120 len 24576 data: 2 2 2 2 2 2 21/30 Write block 429 len 8192 data: 2 2 22/30 Write block 41 len 36864 data: 2 2 2 2 2 2 2 2 2 23/30 Read block 276 len 4096 24/30 Write block 286 len 16384 data: 2 2 2 2 25/30 Read block 135 len 12288 26/30 Write block 547 len 20480 data: 3 3 3 3 3 27/30 Write block 456 len 40960 data: 2 2 2 2 2 3 2 2 2 2 28/30 Write block 349 len 4096 data: 2 29/30 Read block 348 len 28672 30/30 Read block 401 len 12288 ---------------------------------------------------------------- Crucible gen:4 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Sent Done false 1001 Acked Write 6 Done Sent Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 9 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Read 1 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 2 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Write 10 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 10 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Write 9 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:05.145820165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1216} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 1 1 1 1 1 1 T F T <--- 3 060-079 1 1 1 1 1 1 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 1 1 1 1 1 1 T F T <--- 7 140-159 1 1 1 1 1 1 T F T <--- 8 160-179 1 1 1 1 1 1 T F T <--- 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 1 1 1 1 1 1 T F T <--- 12 240-259 2 2 2 2 2 2 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 2 2 2 3 3 3 T F T <--- 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 1 1 1 1 1 1 T F T <--- 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 2 2 2 3 3 3 T F T <--- 22 440-459 1 1 1 1 1 1 T F T <--- 23 460-479 1 1 1 1 1 1 T F T <--- 24 480-499 3 3 3 4 4 4 F F F 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 2 2 2 2 2 2 T F T <--- 28 560-579 1 1 1 1 1 1 T F T <--- 29 580-599 1 1 1 1 1 1 F F F Max gen: 3, Max flush: 4 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 2 Verifying data now No change after verify Loop: 2 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 1 1 1 1 1 F F F 1 020-039 3 3 3 4 4 4 F F F 2 040-059 4 4 4 5 5 5 F F F 3 060-079 1 1 1 1 1 1 F F F 4 080-099 2 2 2 3 3 3 F F F 5 100-119 1 1 1 1 1 1 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 4 4 4 5 5 5 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 4 4 4 5 5 5 F F F 12 240-259 2 2 2 2 2 2 F F F 13 260-279 2 2 2 2 2 2 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 4 4 5 5 5 F F F 18 360-379 1 1 1 1 1 1 F F F 19 380-399 3 3 3 4 4 4 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 4 4 4 5 5 5 F F F 24 480-499 3 3 3 4 4 4 F F F 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 4 4 4 5 5 5 F F F 28 560-579 4 4 4 5 5 5 F F F 29 580-599 1 1 1 1 1 1 F F F Max gen: 4, Max flush: 5 Begin loop 3 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.460863458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:09.471548937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.471599975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482022868Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482331134Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482371957Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482384201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482401784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} {"msg":"Crucible 7f56d7de-b831-4420-b0ab-0a67710a50ff has session id: b471d021-d97c-4112-b2cd-bb7975c46f65","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482752751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Upstairs opts: Upstairs UUID: 7f56d7de-b831-4420-b0ab-0a67710a50ff, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.482786027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Crucible stats registered with UUID: 7f56d7de-b831-4420-b0ab-0a67710a50ff","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.48280155Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:09.482815034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483025543Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483063917Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.48307947Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483189782Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"task":"crutest"} {"msg":"The guest has requested activation with gen:6","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483206695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} {"msg":"7f56d7de-b831-4420-b0ab-0a67710a50ff active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483228055Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.48324098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"0","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483255154Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"1","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483269538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"2","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483283342Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483298225Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483312619Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"ds_connection connected from Ok(127.0.0.1:49766)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483351802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"1","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"ds_connection connected from Ok(127.0.0.1:46895)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483375781Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"2","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"ds_connection connected from Ok(127.0.0.1:34182)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.483390935Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"io task","client":"0","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484084234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"1","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484111363Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"2","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484249723Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"0","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.48446416Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484481742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[0]R flush_numbers[0..12]: [1, 4, 5, 1, 3, 1, 5, 5, 5, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484627849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[0]R generation[0..12]: [1, 3, 4, 1, 2, 1, 4, 4, 4, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484644542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484661054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[1]R flush_numbers[0..12]: [1, 4, 5, 1, 3, 1, 5, 5, 5, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484676168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[1]R generation[0..12]: [1, 3, 4, 1, 2, 1, 4, 4, 4, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484690892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484707164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[2]R flush_numbers[0..12]: [1, 4, 5, 1, 3, 1, 5, 5, 5, 1, 1, 5]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484722138Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[2]R generation[0..12]: [1, 3, 4, 1, 2, 1, 4, 4, 4, 1, 1, 4]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484736892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484752895Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Max found gen is 5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484766299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Generation requested: 6 >= found:5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.48478646Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Next flush: 6","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484801973Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484815188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"":"downstairs","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484828982Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484841436Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"0","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.4848551Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"1","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:49:09.484869274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"client":"2","":"downstairs","session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484886167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484899281Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"7f56d7de-b831-4420-b0ab-0a67710a50ff is now active with session: b471d021-d97c-4112-b2cd-bb7975c46f65","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484913995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484926409Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224,"session_id":"b471d021-d97c-4112-b2cd-bb7975c46f65"} {"msg":"The guest has finished waiting for activation with:6","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484939494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} {"msg":"Activated sub_volume 7f56d7de-b831-4420-b0ab-0a67710a50ff","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:09.484950989Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 24 len 12288 data: 2 2 2 02/30 Write block 85 len 8192 data: 2 2 03/30 Write block 554 len 32768 data: 3 3 3 2 2 2 2 2 04/30 Write block 276 len 16384 data: 2 2 2 2 05/30 Write block 341 len 16384 data: 2 2 2 2 06/30 Read block 550 len 24576 07/30 Flush 08/30 Write block 5 len 24576 data: 2 2 2 2 2 2 09/30 Write block 378 len 24576 data: 2 2 2 3 3 2 10/30 Read block 407 len 16384 11/30 Read block 180 len 28672 12/30 Read block 265 len 40960 13/30 Write block 369 len 16384 data: 2 2 2 2 14/30 Read block 8 len 20480 15/30 Read block 43 len 32768 16/30 Write block 269 len 20480 data: 2 2 2 2 2 17/30 Flush 18/30 Write block 239 len 16384 data: 2 2 2 2 19/30 Read block 503 len 4096 20/30 Flush 21/30 Write block 74 len 16384 data: 2 2 2 2 22/30 Read block 27 len 24576 23/30 Write block 579 len 12288 data: 2 2 2 24/30 Read block 132 len 24576 25/30 Write block 112 len 20480 data: 2 2 2 2 2 26/30 Read block 214 len 12288 27/30 Write block 488 len 40960 data: 2 2 2 2 2 2 2 3 3 3 28/30 Write block 56 len 12288 data: 2 2 2 29/30 Write block 6 len 24576 data: 3 3 3 3 3 2 30/30 Read block 188 len 36864 ---------------------------------------------------------------- Crucible gen:6 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Write 2 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 4 Sent Done Done false 1004 Acked Write 4 Sent Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 6 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Read 4 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 4 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 4 Sent Done Done false 1021 Acked Read 6 Sent Done Done false 1022 Acked Write 3 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Write 10 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 6 Sent Done Done false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:09.496728604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1224} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 1 6 6 1 7 7 F T T <--- 1 020-039 3 6 6 4 6 6 F F F <--- 2 040-059 4 4 4 5 5 5 F T T <--- 3 060-079 1 1 1 1 1 1 F T T <--- 4 080-099 2 6 6 3 6 6 F F F <--- 5 100-119 1 1 1 1 1 1 F T T <--- 6 120-139 4 4 4 5 5 5 F F F 7 140-159 4 4 4 5 5 5 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 4 6 6 5 8 8 F F F <--- 12 240-259 2 6 6 2 8 8 F F F <--- 13 260-279 2 6 6 2 7 7 F F F <--- 14 280-299 4 4 4 5 5 5 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 4 6 6 5 6 6 F F F <--- 18 360-379 1 6 6 1 7 7 F F F <--- 19 380-399 3 6 6 4 7 7 F F F <--- 20 400-419 2 2 2 3 3 3 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 4 4 4 5 5 5 F F F 24 480-499 3 3 3 4 4 4 F T T <--- 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 4 6 6 5 6 6 F F F <--- 28 560-579 4 6 6 5 6 6 F T T <--- 29 580-599 1 1 1 1 1 1 F T T <--- Max gen: 6, Max flush: 8 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 3 Verifying data now No change after verify Loop: 3 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 6 6 6 6 6 6 F F F 2 040-059 7 7 7 9 9 9 F F F 3 060-079 7 7 7 9 9 9 F F F 4 080-099 6 6 6 6 6 6 F F F 5 100-119 7 7 7 9 9 9 F F F 6 120-139 4 4 4 5 5 5 F F F 7 140-159 4 4 4 5 5 5 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 8 8 8 F F F 12 240-259 6 6 6 8 8 8 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 2 2 2 3 3 3 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 6 6 6 6 6 6 F F F 18 360-379 6 6 6 7 7 7 F F F 19 380-399 6 6 6 7 7 7 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 4 4 4 5 5 5 F F F 23 460-479 4 4 4 5 5 5 F F F 24 480-499 7 7 7 9 9 9 F F F 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 6 6 6 6 6 6 F F F 28 560-579 7 7 7 9 9 9 F F F 29 580-599 7 7 7 9 9 9 F F F Max gen: 7, Max flush: 9 Begin loop 4 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.829254829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841124925Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.84143566Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841476722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841489177Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841510138Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} {"msg":"Crucible 27640ec4-948c-4bff-86be-c0bd9fd9fabb has session id: 3c5b9ddd-d296-4a6b-bed9-dd215bb40d61","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841820613Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Upstairs opts: Upstairs UUID: 27640ec4-948c-4bff-86be-c0bd9fd9fabb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841856788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Crucible stats registered with UUID: 27640ec4-948c-4bff-86be-c0bd9fd9fabb","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.841872541Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:13.841885985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.8420811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.84210625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842121213Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842251567Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"task":"crutest"} {"msg":"The guest has requested activation with gen:8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842269429Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} {"msg":"27640ec4-948c-4bff-86be-c0bd9fd9fabb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842304943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842318807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"0","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842332651Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"1","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842346585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"2","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842360409Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842376792Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842391266Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"ds_connection connected from Ok(127.0.0.1:37629)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842494871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"1","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"ds_connection connected from Ok(127.0.0.1:48677)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842522339Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"0","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"ds_connection connected from Ok(127.0.0.1:56797)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.842609991Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"io task","client":"2","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843211351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"2","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843239259Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"0","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843369242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"1","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843580751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843625791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[0]R flush_numbers[0..12]: [9, 6, 9, 9, 6, 9, 5, 5, 5, 1, 1, 8]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843729496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[0]R generation[0..12]: [7, 6, 7, 7, 6, 7, 4, 4, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843746169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843762952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[1]R flush_numbers[0..12]: [9, 6, 9, 9, 6, 9, 5, 5, 5, 1, 1, 8]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843777995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[1]R generation[0..12]: [7, 6, 7, 7, 6, 7, 4, 4, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843793348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.84381285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[2]R flush_numbers[0..12]: [9, 6, 9, 9, 6, 9, 5, 5, 5, 1, 1, 8]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843828273Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[2]R generation[0..12]: [7, 6, 7, 7, 6, 7, 4, 4, 4, 1, 1, 6]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843843267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.84385954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Max found gen is 8","v":0,"name":"crucibleWait for a query_work_queue command to finish before sending IO ","level":30,"time":"2026-03-12T22:49:13.843879011Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Generation requested: 8 >= found:8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843893465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Next flush: 10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843906709Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843920223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"":"downstairs","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843934637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843947242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"0","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:49:13.843961605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"1","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843978498Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"client":"2","":"downstairs","session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.843992552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.844006056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"27640ec4-948c-4bff-86be-c0bd9fd9fabb is now active with session: 3c5b9ddd-d296-4a6b-bed9-dd215bb40d61","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.84402075Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.844034554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232,"session_id":"3c5b9ddd-d296-4a6b-bed9-dd215bb40d61"} {"msg":"The guest has finished waiting for activation with:8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.844048158Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} {"msg":"Activated sub_volume 27640ec4-948c-4bff-86be-c0bd9fd9fabb","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:13.844059633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 440 len 36864 data: 2 2 2 2 2 2 2 2 2 02/30 Flush 03/30 Write block 342 len 24576 data: 3 3 3 2 2 2 04/30 Read block 285 len 40960 05/30 Write block 75 len 36864 data: 3 3 3 2 2 2 2 2 2 06/30 Read block 248 len 4096 07/30 Read block 465 len 36864 08/30 Write block 106 len 24576 data: 2 2 2 2 2 2 09/30 Read block 516 len 32768 10/30 Read block 326 len 20480 11/30 Read block 110 len 40960 12/30 Read block 53 len 40960 13/30 Read block 6 len 12288 14/30 Write block 29 len 24576 data: 2 3 3 3 3 3 15/30 Flush 16/30 Write block 229 len 40960 data: 3 3 3 3 3 3 3 3 3 2 17/30 Read block 364 len 8192 18/30 Write block 255 len 16384 data: 3 3 3 3 19/30 Write block 365 len 24576 data: 2 2 2 2 3 3 20/30 Write block 56 len 20480 data: 3 3 3 2 2 21/30 Write block 239 len 12288 data: 3 3 3 22/30 Read block 128 len 40960 23/30 Write block 465 len 24576 data: 3 2 2 2 2 2 24/30 Read block 581 len 8192 25/30 Write block 148 len 16384 data: 3 3 3 3 26/30 Read block 91 len 20480 27/30 Read block 490 len 28672 28/30 Write block 122 len 32768 data: 3 3 3 3 2 2 2 2 29/30 Write block 482 len 32768 data: 2 2 2 2 2 2 3 3 30/30 Write block 312 len 24576 data: 3 3 3 3 3 2 ---------------------------------------------------------------- Crucible gen:8 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 6 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Write 9 Done Sent Done false 1005 Acked Read 1 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Write 6 Done Sent Done false 1008 Acked Read 8 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Read 10 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Read 3 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 10 Done Sent Done false 1016 Acked Read 2 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Write 6 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Write 3 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Write 6 Done Sent Done false 1023 Acked Read 2 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Read 7 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 26 1 26 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) Some(JobId(1001)) Some(JobId(1014)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:13.856984521Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1232} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 8 6 8 11 6 11 F F F <--- 2 040-059 7 7 7 9 9 9 T F T <--- 3 060-079 8 7 8 11 9 11 T F T <--- 4 080-099 8 6 8 11 6 11 F F F <--- 5 100-119 8 7 8 11 9 11 F F F <--- 6 120-139 4 4 4 5 5 5 T F T <--- 7 140-159 4 4 4 5 5 5 T F T <--- 8 160-179 4 4 4 5 5 5 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 6 6 6 8 8 8 T F T <--- 12 240-259 6 6 6 8 8 8 T F T <--- 13 260-279 6 6 6 7 7 7 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 2 2 2 3 3 3 T F T <--- 16 320-339 2 2 2 3 3 3 F F F 17 340-359 8 6 8 11 6 11 F T F <--- 18 360-379 6 6 6 7 7 7 T F T <--- 19 380-399 6 6 6 7 7 7 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 4 4 4 5 5 5 T F T <--- 24 480-499 7 7 7 9 9 9 T F T <--- 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 6 6 6 6 6 6 F F F 28 560-579 7 7 7 9 9 9 F F F 29 580-599 7 7 7 9 9 9 F F F Max gen: 8, Max flush: 11 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 4 Verifying data now No change after verify Loop: 4 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 8 8 8 11 11 11 F F F 2 040-059 9 9 9 12 12 12 F F F 3 060-079 9 9 9 12 12 12 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 8 8 8 11 11 11 F F F 6 120-139 9 9 9 12 12 12 F F F 7 140-159 9 9 9 12 12 12 F F F 8 160-179 4 4 4 5 5 5 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 9 9 9 12 12 12 F F F 12 240-259 9 9 9 12 12 12 F F F 13 260-279 6 6 6 7 7 7 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 9 9 9 12 12 12 F F F 16 320-339 2 2 2 3 3 3 F F F 17 340-359 8 8 8 11 11 11 F F F 18 360-379 9 9 9 12 12 12 F F F 19 380-399 6 6 6 7 7 7 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 4 4 4 5 5 5 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 9 9 9 12 12 12 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 3 3 3 4 4 4 F F F 26 520-539 2 2 2 3 3 3 F F F 27 540-559 6 6 6 6 6 6 F F F 28 560-579 7 7 7 9 9 9 F F F 29 580-599 7 7 7 9 9 9 F F F Max gen: 9, Max flush: 12 Begin loop 5 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.179542597Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.191242657Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.191593585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.191632568Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.191644722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.191662525Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} {"msg":"Crucible 079acbe7-1d12-42bf-8f8b-1c2a4e578663 has session id: 4a0fc48b-4dc9-483f-bc15-e0a442783a33","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192013563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Upstairs opts: Upstairs UUID: 079acbe7-1d12-42bf-8f8b-1c2a4e578663, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192062411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Crucible stats registered with UUID: 079acbe7-1d12-42bf-8f8b-1c2a4e578663","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192078974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:18.192092398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192262824Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192298849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192317081Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192495474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"task":"crutest"} {"msg":"The guest has requested activation with gen:10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192509837Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} {"msg":"079acbe7-1d12-42bf-8f8b-1c2a4e578663 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192523352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192535816Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"0","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.19254975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"1","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192564174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"2","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192577738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192592162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192606336Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"ds_connection connected from Ok(127.0.0.1:39357)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192633624Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"2","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"ds_connection connected from Ok(127.0.0.1:63574)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192648537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"1","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"ds_connection connected from Ok(127.0.0.1:33767)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.192762078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"io task","client":"0","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193319836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"1","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193359219Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"0","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193433477Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"2","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193683979Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193723032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[0]R flush_numbers[0..12]: [9, 11, 12, 12, 11, 11, 12, 12, 5, 1, 1, 12]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193799049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[0]R generation[0..12]: [7, 8, 9, 9, 8, 8, 9, 9, 4, 1, 1, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193815372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193831834Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[1]R flush_numbers[0..12]: [9, 11, 12, 12, 11, 11, 12, 12, 5, 1, 1, 12]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193847028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[1]R generation[0..12]: [7, 8, 9, 9, 8, 8, 9, 9, 4, 1, 1, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193862101Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193878434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[2]R flush_numbers[0..12]: [9, 11, 12, 12, 11, 11, 12, 12, 5, 1, 1, 12]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193893528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[2]R generation[0..12]: [7, 8, 9, 9, 8, 8, 9, 9, 4, 1, 1, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193908351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193924464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Max found gen is 10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193937949Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Generation requested: 10 >= found:10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.19395744Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Next flush: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193970874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.193986398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"":"downstairs","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194000591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194013276Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"0","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.19402734Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"1","":"downstairs","session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194041224Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"client":"2","":"downstairs"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194059206Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.1940727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"079acbe7-1d12-42bf-8f8b-1c2a4e578663 is now active with session: 4a0fc48b-4dc9-483f-bc15-e0a442783a33","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194088033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194101048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240,"session_id":"4a0fc48b-4dc9-483f-bc15-e0a442783a33"} {"msg":"The guest has finished waiting for activation with:10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194114292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} {"msg":"Activated sub_volume 079acbe7-1d12-42bf-8f8b-1c2a4e578663","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:18.194125877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 513 len 36864 data: 2 2 2 2 3 3 3 3 3 02/30 Read block 111 len 20480 03/30 Read block 213 len 28672 04/30 Read block 404 len 12288 05/30 Read block 420 len 24576 06/30 Flush 07/30 Write block 467 len 32768 data: 3 3 3 3 2 2 2 2 08/30 Write block 379 len 12288 data: 3 3 4 09/30 Read block 465 len 28672 10/30 Read block 334 len 32768 11/30 Write block 135 len 28672 data: 2 2 2 2 2 2 2 12/30 Flush 13/30 Write block 260 len 12288 data: 3 3 3 14/30 Write block 301 len 20480 data: 2 2 2 2 2 15/30 Flush 16/30 Write block 422 len 12288 data: 2 2 2 17/30 Flush 18/30 Write block 327 len 40960 data: 2 2 2 3 3 3 3 3 3 3 19/30 Read block 300 len 8192 20/30 Read block 324 len 36864 21/30 Write block 352 len 40960 data: 2 2 2 2 2 2 2 2 2 2 22/30 Flush 23/30 Write block 168 len 32768 data: 2 2 2 2 2 2 2 2 24/30 Read block 477 len 20480 25/30 Read block 162 len 16384 26/30 Flush 27/30 Write block 257 len 40960 data: 4 4 3 4 4 4 2 2 2 2 28/30 Read block 46 len 12288 29/30 Write block 338 len 32768 data: 3 2 2 3 4 4 4 3 30/30 Read block 522 len 36864 ---------------------------------------------------------------- Crucible gen:10 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Sent false 1001 Acked Read 5 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Read 6 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 4 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 3 Done Done Sent false 1028 Acked Write 8 Done Done Sent false 1029 Acked Read 9 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:18.207968808Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1240} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 8 8 8 11 11 11 F F F 2 040-059 9 9 9 12 12 12 F F F 3 060-079 9 9 9 12 12 12 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 8 8 8 11 11 11 F F F 6 120-139 10 10 9 14 14 12 F F F <--- 7 140-159 10 10 9 14 14 12 F F F <--- 8 160-179 10 10 4 18 18 5 F F F <--- 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 9 9 9 12 12 12 F F F 12 240-259 9 9 9 12 12 12 T T F <--- 13 260-279 10 10 6 15 15 7 T T F <--- 14 280-299 4 4 4 5 5 5 F F F 15 300-319 10 10 9 15 15 12 F F F <--- 16 320-339 10 10 2 17 17 3 T T F <--- 17 340-359 10 10 8 17 17 11 T T F <--- 18 360-379 10 10 9 17 17 12 F F F <--- 19 380-399 10 10 6 14 14 7 F F F <--- 20 400-419 2 2 2 3 3 3 F F F 21 420-439 10 10 4 16 16 5 F F F <--- 22 440-459 8 8 8 10 10 10 F F F 23 460-479 10 10 9 14 14 12 F F F <--- 24 480-499 9 9 9 12 12 12 F F F 25 500-519 10 10 3 13 13 4 F F F <--- 26 520-539 10 10 2 13 13 3 F F F <--- 27 540-559 6 6 6 6 6 6 F F F 28 560-579 7 7 7 9 9 9 F F F 29 580-599 7 7 7 9 9 9 F F F Max gen: 10, Max flush: 18 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 5 Verifying data now No change after verify Loop: 5 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 8 8 8 11 11 11 F F F 2 040-059 9 9 9 12 12 12 F F F 3 060-079 9 9 9 12 12 12 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 8 8 8 11 11 11 F F F 6 120-139 10 10 10 14 14 14 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 1 1 1 1 1 1 F F F 10 200-219 1 1 1 1 1 1 F F F 11 220-239 9 9 9 12 12 12 F F F 12 240-259 11 11 11 19 19 19 F F F 13 260-279 11 11 11 19 19 19 F F F 14 280-299 4 4 4 5 5 5 F F F 15 300-319 10 10 10 15 15 15 F F F 16 320-339 11 11 11 19 19 19 F F F 17 340-359 11 11 11 19 19 19 F F F 18 360-379 10 10 10 17 17 17 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 2 2 2 3 3 3 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 10 10 10 14 14 14 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 10 10 10 13 13 13 F F F 27 540-559 6 6 6 6 6 6 F F F 28 560-579 7 7 7 9 9 9 F F F 29 580-599 7 7 7 9 9 9 F F F Max gen: 11, Max flush: 19 Begin loop 6 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.522299139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534011793Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534334273Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534373796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534386271Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534404063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} {"msg":"Crucible a9878eb9-23a8-479a-aa5f-8b3d42165c3f has session id: fbc6f2a7-419b-496a-b71e-60ade03ea978","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534769215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Upstairs opts: Upstairs UUID: a9878eb9-23a8-479a-aa5f-8b3d42165c3f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534795203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Crucible stats registered with UUID: a9878eb9-23a8-479a-aa5f-8b3d42165c3f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.534809447Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:22.534822662Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.53501168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535048064Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535063817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535200788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"task":"crutest"} {"msg":"The guest has requested activation with gen:12","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535215451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} {"msg":"a9878eb9-23a8-479a-aa5f-8b3d42165c3f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535231374Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535244069Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"0","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535258073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"1","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535272097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"2","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535285731Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535300095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535322815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"ds_connection connected from Ok(127.0.0.1:45892)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535370154Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"1","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"ds_connection connected from Ok(127.0.0.1:48510)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535397762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"2","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"ds_connection connected from Ok(127.0.0.1:38392)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.535413705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"io task","client":"0","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536140221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"0","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536168918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"1","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536189359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"2","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536490479Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.5365348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[0]R flush_numbers[0..12]: [9, 11, 12, 12, 11, 11, 14, 14, 18, 1, 1, 12]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536588126Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[0]R generation[0..12]: [7, 8, 9, 9, 8, 8, 10, 10, 10, 1, 1, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536604649Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536620952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[1]R flush_numbers[0..12]: [9, 11, 12, 12, 11, 11, 14, 14, 18, 1, 1, 12]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536636076Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[1]R generation[0..12]: [7, 8, 9, 9, 8, 8, 10, 10, 10, 1, 1, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536651349Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536670281Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[2]R flush_numbers[0..12]: [9, 11, 12, 12, 11, 11, 14, 14, 18, 1, 1, 12]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536685424Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[2]R generation[0..12]: [7, 8, 9, 9, 8, 8, 10, 10, 10, 1, 1, 9]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536700018Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536716191Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Max found gen is 12","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536729575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Generation requested: 12 >= found:12","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536750226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Next flush: 20","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.53676366Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536777035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"":"downstairs","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536790909Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536803483Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"0","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536817467Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"1","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:49:22.536831201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"client":"2","":"downstairs","session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536847514Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536860748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"a9878eb9-23a8-479a-aa5f-8b3d42165c3f is now active with session: fbc6f2a7-419b-496a-b71e-60ade03ea978","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536875532Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536889766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248,"session_id":"fbc6f2a7-419b-496a-b71e-60ade03ea978"} {"msg":"The guest has finished waiting for activation with:12","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.53690279Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} {"msg":"Activated sub_volume a9878eb9-23a8-479a-aa5f-8b3d42165c3f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:22.536914165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 35 len 8192 data: 3 3 02/30 Write block 64 len 36864 data: 2 2 2 2 2 2 2 2 2 03/30 Read block 586 len 8192 04/30 Write block 363 len 8192 data: 2 2 05/30 Write block 258 len 40960 data: 5 4 5 5 5 3 3 3 3 2 06/30 Write block 539 len 8192 data: 2 2 07/30 Read block 374 len 8192 08/30 Write block 401 len 20480 data: 2 3 3 3 3 09/30 Write block 209 len 16384 data: 2 2 2 2 10/30 Write block 45 len 24576 data: 3 3 3 3 3 2 11/30 Read block 124 len 20480 12/30 Flush 13/30 Write block 586 len 28672 data: 2 2 2 2 2 2 2 14/30 Write block 282 len 32768 data: 2 2 2 2 3 3 3 3 15/30 Flush 16/30 Flush 17/30 Write block 238 len 36864 data: 3 4 4 4 3 2 2 2 2 18/30 Write block 189 len 24576 data: 2 2 2 2 2 2 19/30 Read block 291 len 20480 20/30 Read block 13 len 20480 21/30 Read block 567 len 28672 22/30 Read block 507 len 20480 23/30 Read block 565 len 24576 24/30 Read block 34 len 12288 25/30 Read block 207 len 12288 26/30 Read block 288 len 8192 27/30 Write block 120 len 16384 data: 3 3 4 4 28/30 Write block 347 len 16384 data: 3 2 3 2 29/30 Write block 211 len 16384 data: 3 3 2 2 30/30 Write block 592 len 24576 data: 3 2 2 2 2 2 ---------------------------------------------------------------- Crucible gen:12 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Done Sent false 1003 Acked Write 2 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 2 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:22.547599884Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1248} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 12 12 8 20 20 11 F F T <--- 2 040-059 12 12 9 20 20 12 F F F <--- 3 060-079 12 12 9 20 20 12 F F T <--- 4 080-099 8 8 8 11 11 11 F F F 5 100-119 8 8 8 11 11 11 F F F 6 120-139 10 10 10 14 14 14 T T F <--- 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 1 1 1 1 1 1 T T F <--- 10 200-219 12 12 1 20 20 1 T T F <--- 11 220-239 9 9 9 12 12 12 T T F <--- 12 240-259 12 12 11 20 20 19 T T F <--- 13 260-279 12 12 11 20 20 19 F F F <--- 14 280-299 12 12 4 21 21 5 F F F <--- 15 300-319 10 10 10 15 15 15 F F F 16 320-339 11 11 11 19 19 19 F F F 17 340-359 11 11 11 19 19 19 T T F <--- 18 360-379 12 12 10 20 20 17 F F F <--- 19 380-399 10 10 10 14 14 14 F F F 20 400-419 12 12 2 20 20 3 F F F <--- 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 10 10 10 14 14 14 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 12 12 10 20 20 13 F F F <--- 27 540-559 12 12 6 20 20 6 F F F <--- 28 560-579 7 7 7 9 9 9 F F F 29 580-599 12 12 7 21 21 9 T T F <--- Max gen: 12, Max flush: 21 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 6 Verifying data now No change after verify Loop: 6 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 12 12 12 20 20 20 F F F 3 060-079 12 12 12 20 20 20 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 8 8 8 11 11 11 F F F 6 120-139 13 13 13 22 22 22 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 13 13 13 22 22 22 F F F 10 200-219 13 13 13 22 22 22 F F F 11 220-239 13 13 13 22 22 22 F F F 12 240-259 13 13 13 22 22 22 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 12 12 12 21 21 21 F F F 15 300-319 10 10 10 15 15 15 F F F 16 320-339 11 11 11 19 19 19 F F F 17 340-359 13 13 13 22 22 22 F F F 18 360-379 12 12 12 20 20 20 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 8 8 8 10 10 10 F F F 23 460-479 10 10 10 14 14 14 F F F 24 480-499 9 9 9 12 12 12 F F F 25 500-519 10 10 10 13 13 13 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 7 7 7 9 9 9 F F F 29 580-599 13 13 13 22 22 22 F F F Max gen: 13, Max flush: 22 Begin loop 7 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.87403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.885693225Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.88600558Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886061596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.88607506Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886093032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} {"msg":"Crucible 143ba553-d18f-407b-a755-8f35ae59bf3b has session id: 777eb115-38c6-4230-9b60-09ba8014c6c7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886448588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Upstairs opts: Upstairs UUID: 143ba553-d18f-407b-a755-8f35ae59bf3b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886482463Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Crucible stats registered with UUID: 143ba553-d18f-407b-a755-8f35ae59bf3b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886497946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:26.886511131Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886679498Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886705217Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.8867198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886852303Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"task":"crutest"} {"msg":"The guest has requested activation with gen:14","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886871784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} {"msg":"143ba553-d18f-407b-a755-8f35ae59bf3b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886895044Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886907789Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"0","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886921783Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"1","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886935717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"2","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886949241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886963575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.886977948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50384)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.887027757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"0","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:33858)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.887145616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"1","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62845)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.887173224Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"io task","client":"2","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.887790636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"0","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.887818974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"1","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.88801264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"2","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888180337Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888224118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[0]R flush_numbers[0..12]: [9, 20, 20, 20, 11, 11, 22, 14, 18, 22, 22, 22]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888354042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[0]R generation[0..12]: [7, 12, 12, 12, 8, 8, 13, 10, 10, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888370755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888387188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[1]R flush_numbers[0..12]: [9, 20, 20, 20, 11, 11, 22, 14, 18, 22, 22, 22]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888402491Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[1]R generation[0..12]: [7, 12, 12, 12, 8, 8, 13, 10, 10, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888417824Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888434227Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[2]R flush_numbers[0..12]: [9, 20, 20, 20, 11, 11, 22, 14, 18, 22, 22, 22]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888449361Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[2]R generation[0..12]: [7, 12, 12, 12, 8, 8, 13, 10, 10, 13, 13, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888464604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888480837Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Max found gen is 14","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888494151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 14 >= found:14","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888512463Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Next flush: 23","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888526068Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888539452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"":"downstairs","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888555415Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888567999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"0","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888581743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"1","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:49:26.888595377Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"client":"2","":"downstairs","session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.88861186Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888625354Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"143ba553-d18f-407b-a755-8f35ae59bf3b is now active with session: 777eb115-38c6-4230-9b60-09ba8014c6c7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888640428Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888653182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256,"session_id":"777eb115-38c6-4230-9b60-09ba8014c6c7"} {"msg":"The guest has finished waiting for activation with:14","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888666367Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} {"msg":"Activated sub_volume 143ba553-d18f-407b-a755-8f35ae59bf3b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:26.888677602Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 20480 data: 2 3 3 3 2 02/30 Write block 459 len 24576 data: 3 3 4 3 3 3 03/30 Read block 263 len 24576 04/30 Write block 467 len 36864 data: 4 4 4 4 3 3 3 3 2 05/30 Flush 06/30 Write block 479 len 20480 data: 2 2 2 3 3 07/30 Read block 495 len 8192 08/30 Read block 416 len 32768 09/30 Read block 144 len 24576 10/30 Write block 104 len 4096 data: 2 11/30 Write block 284 len 36864 data: 3 3 4 4 4 4 2 2 2 12/30 Write block 186 len 24576 data: 2 2 2 3 3 3 13/30 Write block 204 len 12288 data: 2 2 2 14/30 Flush 15/30 Write block 497 len 28672 data: 4 3 3 3 2 2 2 16/30 Read block 484 len 20480 17/30 Write block 185 len 12288 data: 2 3 3 18/30 Read block 478 len 12288 19/30 Write block 464 len 32768 data: 4 4 3 5 5 5 5 4 20/30 Read block 268 len 8192 21/30 Read block 502 len 8192 22/30 Read block 28 len 24576 23/30 Read block 249 len 12288 24/30 Write block 196 len 16384 data: 2 2 2 2 25/30 Read block 126 len 4096 26/30 Read block 515 len 40960 27/30 Write block 15 len 16384 data: 2 2 2 2 28/30 Write block 589 len 24576 data: 3 3 3 4 3 3 29/30 Write block 361 len 16384 data: 3 2 3 3 30/30 Read block 402 len 4096 ---------------------------------------------------------------- Crucible gen:14 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 6 Done Done Done false 1002 Acked Read 6 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 5 Done Done Sent false 1006 Acked Read 2 Done Done Sent false 1007 Acked Read 8 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 4 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 28 30 Done 30 28 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:26.900416604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1256} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 7 7 7 9 9 9 T T F <--- 1 020-039 12 12 12 20 20 20 F F F 2 040-059 12 12 12 20 20 20 F F F 3 060-079 12 12 12 20 20 20 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 14 14 8 24 24 11 F F F <--- 6 120-139 13 13 13 22 22 22 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 14 14 13 24 24 22 T T F <--- 10 200-219 14 14 13 24 24 22 F F F <--- 11 220-239 13 13 13 22 22 22 F F F 12 240-259 13 13 13 22 22 22 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 14 14 12 24 24 21 F F F <--- 15 300-319 10 10 10 15 15 15 F F F 16 320-339 11 11 11 19 19 19 F F F 17 340-359 13 13 13 22 22 22 F F F 18 360-379 12 12 12 20 20 20 T T F <--- 19 380-399 10 10 10 14 14 14 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 14 14 8 23 23 10 F F T <--- 23 460-479 14 14 10 24 24 14 T T T <--- 24 480-499 14 14 9 24 24 12 T T F <--- 25 500-519 10 10 10 13 13 13 T T F <--- 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 7 23 23 9 F F T <--- 29 580-599 14 14 13 23 23 22 T T T <--- Max gen: 14, Max flush: 24 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 7 Verifying data now No change after verify Loop: 7 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 15 15 15 25 25 25 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 12 12 12 20 20 20 F F F 3 060-079 12 12 12 20 20 20 F F F 4 080-099 8 8 8 11 11 11 F F F 5 100-119 14 14 14 24 24 24 F F F 6 120-139 13 13 13 22 22 22 F F F 7 140-159 10 10 10 14 14 14 F F F 8 160-179 10 10 10 18 18 18 F F F 9 180-199 15 15 15 25 25 25 F F F 10 200-219 14 14 14 24 24 24 F F F 11 220-239 13 13 13 22 22 22 F F F 12 240-259 13 13 13 22 22 22 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 10 10 10 15 15 15 F F F 16 320-339 11 11 11 19 19 19 F F F 17 340-359 13 13 13 22 22 22 F F F 18 360-379 15 15 15 25 25 25 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 12 12 12 20 20 20 F F F 21 420-439 10 10 10 16 16 16 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 15 15 15 25 25 25 F F F 25 500-519 15 15 15 25 25 25 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 15 15 15 25 25 25 F F F Max gen: 15, Max flush: 25 Begin loop 8 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.215678222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.227416675Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.227735557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.22777492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.227787454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.227805387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} {"msg":"Crucible 5f5770e7-a454-4732-a565-acaeab4f2975 has session id: 3dc89d95-ba97-4097-bab8-aef8e91aee52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228200955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Upstairs opts: Upstairs UUID: 5f5770e7-a454-4732-a565-acaeab4f2975, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228231932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Crucible stats registered with UUID: 5f5770e7-a454-4732-a565-acaeab4f2975","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228246865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:31.22825995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228440331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.22846613Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228481034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228624422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"task":"crutest"} {"msg":"The guest has requested activation with gen:16","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228641434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} {"msg":"5f5770e7-a454-4732-a565-acaeab4f2975 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228658027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228670891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"0","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228685045Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"1","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228699339Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"2","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228713163Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228727977Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228751277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"ds_connection connected from Ok(127.0.0.1:62100)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.22876659Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"0","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"ds_connection connected from Ok(127.0.0.1:43136)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228781753Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"2","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"ds_connection connected from Ok(127.0.0.1:56238)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.228796447Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"io task","client":"1","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.229493335Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"0","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.229521623Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"1","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.229565354Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"2","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.229867973Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.229915382Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[0]R flush_numbers[0..12]: [25, 20, 20, 20, 11, 24, 22, 14, 18, 25, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.22998859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[0]R generation[0..12]: [15, 12, 12, 12, 8, 14, 13, 10, 10, 15, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230005613Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230022356Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[1]R flush_numbers[0..12]: [25, 20, 20, 20, 11, 24, 22, 14, 18, 25, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230038129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[1]R generation[0..12]: [15, 12, 12, 12, 8, 14, 13, 10, 10, 15, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230053492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230070075Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[2]R flush_numbers[0..12]: [25, 20, 20, 20, 11, 24, 22, 14, 18, 25, 24, 22]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230087997Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[2]R generation[0..12]: [15, 12, 12, 12, 8, 14, 13, 10, 10, 15, 14, 13]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230103411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230119594Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Max found gen is 16","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230133158Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Generation requested: 16 >= found:16","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230153749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Next flush: 26","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230166963Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230180607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"":"downstairs","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230194431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230207096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"0","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.23022104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"1","":"downstairs","session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230234764Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230251916Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.2302653Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"5f5770e7-a454-4732-a565-acaeab4f2975 is now active with session: 3dc89d95-ba97-4097-bab8-aef8e91aee52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230280414Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230292918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264,"session_id":"3dc89d95-ba97-4097-bab8-aef8e91aee52"} {"msg":"The guest has finished waiting for activation with:16","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230308082Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} {"msg":"Activated sub_volume 5f5770e7-a454-4732-a565-acaeab4f2975","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:31.230319337Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 431 len 20480 data: 2 2 2 2 2 02/30 Read block 325 len 8192 03/30 Read block 253 len 36864 04/30 Write block 47 len 40960 data: 4 4 4 3 2 2 2 2 2 4 05/30 Write block 586 len 12288 data: 3 3 3 06/30 Write block 171 len 24576 data: 3 3 3 3 3 2 07/30 Write block 238 len 24576 data: 4 5 5 5 4 3 08/30 Flush 09/30 Write block 192 len 24576 data: 3 3 3 2 3 3 10/30 Write block 2 len 4096 data: 2 11/30 Flush 12/30 Write block 357 len 20480 data: 3 3 3 3 4 13/30 Read block 308 len 36864 14/30 Write block 308 len 20480 data: 3 3 3 3 4 15/30 Read block 74 len 16384 16/30 Flush 17/30 Write block 409 len 24576 data: 3 2 2 2 2 2 18/30 Read block 560 len 4096 19/30 Write block 95 len 12288 data: 3 3 2 20/30 Read block 34 len 36864 21/30 Write block 413 len 12288 data: 3 3 2 22/30 Write block 193 len 8192 data: 4 4 23/30 Read block 390 len 12288 24/30 Read block 143 len 28672 25/30 Read block 61 len 40960 26/30 Read block 554 len 20480 27/30 Write block 132 len 36864 data: 2 2 2 3 3 3 3 3 3 28/30 Write block 330 len 12288 data: 4 4 4 29/30 Read block 106 len 12288 30/30 Read block 463 len 20480 ---------------------------------------------------------------- Crucible gen:16 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 2 Done Done Done false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 10 Done Done Sent false 1004 Acked Write 3 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Read 9 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 4 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Write 3 Done Done Sent false 1019 Acked Read 9 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 28 29 Done 29 30 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) Some(JobId(1015)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:31.242230675Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1264} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 15 27 27 25 F F F <--- 1 020-039 12 12 12 20 20 20 F F F 2 040-059 16 16 12 26 26 20 F F F <--- 3 060-079 12 12 12 20 20 20 F F F 4 080-099 8 8 8 11 11 11 T T F <--- 5 100-119 14 14 14 24 24 24 F F F 6 120-139 13 13 13 22 22 22 T T F <--- 7 140-159 10 10 10 14 14 14 T T F <--- 8 160-179 16 16 10 26 26 18 F F F <--- 9 180-199 16 16 15 27 27 25 T T F <--- 10 200-219 14 14 14 24 24 24 F F F 11 220-239 16 16 13 26 26 22 F F F <--- 12 240-259 16 16 13 26 26 22 F F F <--- 13 260-279 12 12 12 20 20 20 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 16 16 10 28 28 15 F F F <--- 16 320-339 11 11 11 19 19 19 T T F <--- 17 340-359 16 16 13 28 28 22 F F F <--- 18 360-379 16 16 15 28 28 25 F F F <--- 19 380-399 10 10 10 14 14 14 F F F 20 400-419 12 12 12 20 20 20 T T F <--- 21 420-439 16 16 10 26 26 16 F F T <--- 22 440-459 14 14 14 23 23 23 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 15 15 15 25 25 25 F F F 25 500-519 15 15 15 25 25 25 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 16 16 15 26 26 25 F F F <--- Max gen: 16, Max flush: 28 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 8 Verifying data now No change after verify Loop: 8 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 16 16 16 26 26 26 F F F 3 060-079 12 12 12 20 20 20 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 14 14 14 24 24 24 F F F 6 120-139 17 17 17 29 29 29 F F F 7 140-159 17 17 17 29 29 29 F F F 8 160-179 16 16 16 26 26 26 F F F 9 180-199 17 17 17 29 29 29 F F F 10 200-219 14 14 14 24 24 24 F F F 11 220-239 16 16 16 26 26 26 F F F 12 240-259 16 16 16 26 26 26 F F F 13 260-279 12 12 12 20 20 20 F F F 14 280-299 14 14 14 24 24 24 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 16 16 16 28 28 28 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 17 17 17 29 29 29 F F F 21 420-439 16 16 16 26 26 26 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 15 15 15 25 25 25 F F F 25 500-519 15 15 15 25 25 25 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 16 16 16 26 26 26 F F F Max gen: 17, Max flush: 29 Begin loop 9 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.566928913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.578614659Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.578960329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579001701Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579013646Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579031448Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} {"msg":"Crucible 1b5286a9-84d4-4a57-9595-7a5242cc3174 has session id: f17ca8a7-fca8-41dd-bbc3-460354dd135e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579384355Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Upstairs opts: Upstairs UUID: 1b5286a9-84d4-4a57-9595-7a5242cc3174, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579414642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Crucible stats registered with UUID: 1b5286a9-84d4-4a57-9595-7a5242cc3174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579429496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:35.57944243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579618954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579655838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579671161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579778614Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"task":"crutest"} {"msg":"The guest has requested activation with gen:18","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579795807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} {"msg":"1b5286a9-84d4-4a57-9595-7a5242cc3174 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.57981085Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579823835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"0","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579838209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"1","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579852662Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"2","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579866396Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.57988095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579895314Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"ds_connection connected from Ok(127.0.0.1:43316)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579917974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"1","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"ds_connection connected from Ok(127.0.0.1:38859)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579932958Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"0","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53214)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.579948071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"io task","client":"2","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.580625178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"2","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.580653295Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"0","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.580784578Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"1","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.580972887Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581014899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[0]R flush_numbers[0..12]: [27, 20, 26, 20, 29, 24, 29, 29, 26, 29, 24, 26]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581159026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[0]R generation[0..12]: [16, 12, 16, 12, 17, 14, 17, 17, 16, 17, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581176169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581192812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[1]R flush_numbers[0..12]: [27, 20, 26, 20, 29, 24, 29, 29, 26, 29, 24, 26]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581208215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[1]R generation[0..12]: [16, 12, 16, 12, 17, 14, 17, 17, 16, 17, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581223288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581239471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[2]R flush_numbers[0..12]: [27, 20, 26, 20, 29, 24, 29, 29, 26, 29, 24, 26]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581254605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[2]R generation[0..12]: [16, 12, 16, 12, 17, 14, 17, 17, 16, 17, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581269668Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581285901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Max found gen is 18","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:49:35.581299285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Generation requested: 18 >= found:18","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581318587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Next flush: 30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581331751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581345225Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"":"downstairs","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581361309Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581374163Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"0","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:49:35.581387977Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"1","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.58140426Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"client":"2","":"downstairs","session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581418774Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581432028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"1b5286a9-84d4-4a57-9595-7a5242cc3174 is now active with session: f17ca8a7-fca8-41dd-bbc3-460354dd135e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581447091Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581459506Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272,"session_id":"f17ca8a7-fca8-41dd-bbc3-460354dd135e"} {"msg":"The guest has finished waiting for activation with:18","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.58147243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} {"msg":"Activated sub_volume 1b5286a9-84d4-4a57-9595-7a5242cc3174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:35.581483745Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 141 len 16384 data: 3 2 2 2 02/30 Read block 250 len 36864 03/30 Read block 310 len 4096 04/30 Read block 51 len 8192 05/30 Read block 342 len 20480 06/30 Write block 515 len 12288 data: 3 3 4 07/30 Read block 572 len 28672 08/30 Read block 427 len 8192 09/30 Write block 71 len 8192 data: 3 3 10/30 Write block 50 len 24576 data: 4 3 3 3 3 3 11/30 Write block 259 len 12288 data: 5 6 6 12/30 Write block 259 len 16384 data: 6 7 7 6 13/30 Write block 166 len 24576 data: 2 2 3 3 3 4 14/30 Write block 272 len 24576 data: 3 3 2 2 3 3 15/30 Write block 350 len 12288 data: 3 2 3 16/30 Write block 293 len 20480 data: 2 2 2 3 3 17/30 Read block 386 len 12288 18/30 Flush 19/30 Write block 425 len 8192 data: 2 2 20/30 Write block 286 len 24576 data: 5 5 5 5 3 3 21/30 Read block 414 len 4096 22/30 Write block 584 len 8192 data: 2 2 23/30 Write block 117 len 40960 data: 2 2 2 4 4 5 5 4 4 3 24/30 Read block 328 len 32768 25/30 Read block 199 len 28672 26/30 Flush 27/30 Write block 482 len 4096 data: 4 28/30 Read block 26 len 24576 29/30 Write block 197 len 4096 data: 4 30/30 Write block 417 len 36864 data: 2 2 2 2 2 3 3 3 3 ---------------------------------------------------------------- Crucible gen:18 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Read 5 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 7 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Write 6 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 2 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 1 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Read 8 Done Sent Done false 1024 Acked Read 7 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 1 Done Sent Done false 1027 Acked Read 6 Done Sent Done false 1028 Acked Write 1 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:35.591885827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1272} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 18 16 18 30 26 30 F F F <--- 3 060-079 18 12 18 30 20 30 F F F <--- 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 14 18 31 24 31 F F F <--- 6 120-139 18 17 18 31 29 31 F F F <--- 7 140-159 18 17 18 30 29 30 F T F <--- 8 160-179 18 16 18 30 26 30 F F F <--- 9 180-199 17 17 17 29 29 29 T F T <--- 10 200-219 14 14 14 24 24 24 F F F 11 220-239 16 16 16 26 26 26 F F F 12 240-259 18 16 18 30 26 30 F F F <--- 13 260-279 18 12 18 30 20 30 F F F <--- 14 280-299 18 14 18 31 24 31 F F F <--- 15 300-319 16 16 16 28 28 28 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 18 16 18 30 28 30 F F F <--- 18 360-379 16 16 16 28 28 28 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 17 17 17 29 29 29 T F T <--- 21 420-439 18 16 18 31 26 31 T F T <--- 22 440-459 14 14 14 23 23 23 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 15 15 15 25 25 25 T F T <--- 25 500-519 18 15 18 30 25 30 F F F <--- 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 16 18 31 26 31 F F F <--- Max gen: 18, Max flush: 31 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 9 Verifying data now No change after verify Loop: 9 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 18 18 18 30 30 30 F F F 3 060-079 18 18 18 30 30 30 F F F 4 080-099 17 17 17 29 29 29 F F F 5 100-119 18 18 18 31 31 31 F F F 6 120-139 18 18 18 31 31 31 F F F 7 140-159 18 18 18 30 30 30 F F F 8 160-179 18 18 18 30 30 30 F F F 9 180-199 19 19 19 32 32 32 F F F 10 200-219 14 14 14 24 24 24 F F F 11 220-239 16 16 16 26 26 26 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 18 18 18 30 30 30 F F F 14 280-299 18 18 18 31 31 31 F F F 15 300-319 16 16 16 28 28 28 F F F 16 320-339 17 17 17 29 29 29 F F F 17 340-359 18 18 18 30 30 30 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 10 10 10 14 14 14 F F F 20 400-419 19 19 19 32 32 32 F F F 21 420-439 19 19 19 32 32 32 F F F 22 440-459 14 14 14 23 23 23 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 19 19 19 32 32 32 F F F 25 500-519 18 18 18 30 30 30 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 19, Max flush: 32 Begin loop 10 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.932669233Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.944366853Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.944679418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.944719281Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.944731515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.944749108Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} {"msg":"Crucible afa27a67-dacb-4637-927d-359957e6e21c has session id: ed6c110c-f156-4707-b4ae-73084431ae8f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945095468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Upstairs opts: Upstairs UUID: afa27a67-dacb-4637-927d-359957e6e21c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945139409Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Crucible stats registered with UUID: afa27a67-dacb-4637-927d-359957e6e21c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945155282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:39.945168796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945370089Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945405823Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945421426Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945560666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"task":"crutest"} {"msg":"The guest has requested activation with gen:20","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945577389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} {"msg":"afa27a67-dacb-4637-927d-359957e6e21c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945593532Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945606746Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"0","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.94562097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"1","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945635484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"2","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945649378Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945663591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945678005Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:48685)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.94573555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"2","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:61901)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945752293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"0","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"ds_connection connected from Ok(127.0.0.1:55459)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.945767326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"io task","client":"1","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.946533464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"0","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.946582673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"1","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.946600515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"2","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.946905263Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.946950614Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[0]R flush_numbers[0..12]: [27, 20, 30, 30, 29, 31, 31, 30, 30, 32, 24, 26]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947029909Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[0]R generation[0..12]: [16, 12, 18, 18, 17, 18, 18, 18, 18, 19, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947047752Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947064534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[1]R flush_numbers[0..12]: [27, 20, 30, 30, 29, 31, 31, 30, 30, 32, 24, 26]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947079968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[1]R generation[0..12]: [16, 12, 18, 18, 17, 18, 18, 18, 18, 19, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947095721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947112573Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[2]R flush_numbers[0..12]: [27, 20, 30, 30, 29, 31, 31, 30, 30, 32, 24, 26]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947130626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[2]R generation[0..12]: [16, 12, 18, 18, 17, 18, 18, 18, 18, 19, 14, 16]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947145729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947162812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Max found gen is 20","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947176926Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs",""Wait for a query_work_queue command to finish before sending IO :"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Generation requested: 20 >= found:20","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947198136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Next flush: 33","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.94721215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947226694Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"":"downstairs","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947240978Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947253852Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"0","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947268026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"1","":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947285519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"client":"2","":"downstairs","session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947299243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947312367Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"afa27a67-dacb-4637-927d-359957e6e21c is now active with session: ed6c110c-f156-4707-b4ae-73084431ae8f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.94732736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947340924Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280,"session_id":"ed6c110c-f156-4707-b4ae-73084431ae8f"} {"msg":"The guest has finished waiting for activation with:20","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947355898Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} {"msg":"Activated sub_volume afa27a67-dacb-4637-927d-359957e6e21c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:39.947367713Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 40 len 28672 data: 2 3 3 3 3 4 4 02/30 Read block 64 len 36864 03/30 Write block 90 len 24576 data: 2 2 2 3 3 4 04/30 Read block 311 len 28672 05/30 Write block 123 len 12288 data: 6 5 5 06/30 Write block 286 len 8192 data: 6 6 07/30 Write block 336 len 8192 data: 4 3 08/30 Read block 53 len 36864 09/30 Write block 191 len 40960 data: 4 4 5 5 3 4 5 3 3 2 10/30 Flush 11/30 Write block 297 len 40960 data: 4 2 2 2 3 3 3 3 3 2 12/30 Read block 76 len 24576 13/30 Read block 66 len 32768 14/30 Write block 220 len 24576 data: 2 2 2 2 2 2 15/30 Read block 275 len 4096 16/30 Read block 36 len 16384 17/30 Read block 539 len 4096 18/30 Write block 349 len 28672 data: 4 4 3 4 3 3 3 19/30 Read block 359 len 16384 20/30 Read block 254 len 24576 21/30 Flush 22/30 Write block 396 len 8192 data: 2 2 23/30 Write block 150 len 28672 data: 4 4 2 2 2 2 2 24/30 Write block 14 len 20480 data: 2 3 3 3 3 25/30 Write block 449 len 40960 data: 2 2 2 2 2 2 2 3 3 3 26/30 Read block 408 len 32768 27/30 Write block 435 len 28672 data: 3 2 3 3 3 3 3 28/30 Write block 504 len 20480 data: 2 2 2 2 2 29/30 Read block 499 len 8192 30/30 Write block 483 len 32768 data: 4 3 3 3 3 4 4 3 ---------------------------------------------------------------- Crucible gen:20 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Write 2 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 6 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 4 Done Done Sent false 1016 Acked Read 1 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Write 7 Done Done Sent false 1023 Acked Write 5 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 29 29 5 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:39.960373576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1280} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 16 16 16 27 27 27 T T F <--- 1 020-039 12 12 12 20 20 20 F F F 2 040-059 20 20 18 33 33 30 F F T <--- 3 060-079 18 18 18 30 30 30 F F F 4 080-099 20 20 17 33 33 29 F F T <--- 5 100-119 18 18 18 31 31 31 F F F 6 120-139 20 20 18 33 33 31 F F T <--- 7 140-159 18 18 18 30 30 30 T T F <--- 8 160-179 18 18 18 30 30 30 F F F 9 180-199 20 20 19 33 33 32 F F F <--- 10 200-219 20 20 14 33 33 24 F F F <--- 11 220-239 20 20 16 34 34 26 F F F <--- 12 240-259 18 18 18 30 30 30 F F F 13 260-279 18 18 18 30 30 30 F F F 14 280-299 20 20 18 34 34 31 F F F <--- 15 300-319 20 20 16 34 34 28 F F F <--- 16 320-339 20 20 17 33 33 29 F F F <--- 17 340-359 20 20 18 34 34 30 F F F <--- 18 360-379 16 16 16 28 28 28 F F F 19 380-399 10 10 10 14 14 14 T T F <--- 20 400-419 19 19 19 32 32 32 F F F 21 420-439 19 19 19 32 32 32 T T F <--- 22 440-459 14 14 14 23 23 23 T T F <--- 23 460-479 15 15 15 25 25 25 F F F 24 480-499 19 19 19 32 32 32 T T F <--- 25 500-519 18 18 18 30 30 30 T T F <--- 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 20, Max flush: 34 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 10 Verifying data now No change after verify Loop: 10 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 20 20 20 33 33 33 F F F 3 060-079 18 18 18 30 30 30 F F F 4 080-099 20 20 20 33 33 33 F F F 5 100-119 18 18 18 31 31 31 F F F 6 120-139 20 20 20 33 33 33 F F F 7 140-159 21 21 21 35 35 35 F F F 8 160-179 18 18 18 30 30 30 F F F 9 180-199 20 20 20 33 33 33 F F F 10 200-219 20 20 20 33 33 33 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 18 18 18 30 30 30 F F F 13 260-279 18 18 18 30 30 30 F F F 14 280-299 20 20 20 34 34 34 F F F 15 300-319 20 20 20 34 34 34 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 20 20 20 34 34 34 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 19 19 19 32 32 32 F F F 21 420-439 21 21 21 35 35 35 F F F 22 440-459 21 21 21 35 35 35 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 21 21 21 35 35 35 F F F 26 520-539 12 12 12 20 20 20 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 21, Max flush: 35 Begin loop 11 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.287915641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.299674605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300046794Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300088156Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30010053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300118533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} {"msg":"Crucible 16ba463d-e310-421a-8b30-d007be717fa5 has session id: b17e8f9b-90f7-415e-837d-451267493c5c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.3005155Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Upstairs opts: Upstairs UUID: 16ba463d-e310-421a-8b30-d007be717fa5, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300559971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Crucible stats registered with UUID: 16ba463d-e310-421a-8b30-d007be717fa5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300575724Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:44.300589109Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300760804Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300785643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300800477Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300914408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"task":"crutest"} {"msg":"The guest has requested activation with gen:22","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30093138Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} {"msg":"16ba463d-e310-421a-8b30-d007be717fa5 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300947533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300960308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"0","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300974552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"1","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.300988675Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"2","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301015084Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301029298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301043382Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59926)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301080116Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"2","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33044)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301103636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"0","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"ds_connection connected from Ok(127.0.0.1:33237)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.3012089Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"io task","client":"1","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301822204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"0","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.301851232Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"2","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30196923Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"1","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302245401Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302288502Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[0]R flush_numbers[0..12]: [35, 20, 33, 30, 33, 31, 33, 35, 30, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302350205Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[0]R generation[0..12]: [21, 12, 20, 18, 20, 18, 20, 21, 18, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302366988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302383291Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[1]R flush_numbers[0..12]: [35, 20, 33, 30, 33, 31, 33, 35, 30, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302398744Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[1]R generation[0..12]: [21, 12, 20, 18, 20, 18, 20, 21, 18, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302413938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.3024306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[2]R flush_numbers[0..12]: [35, 20, 33, 30, 33, 31, 33, 35, 30, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302446144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[2]R generation[0..12]: [21, 12, 20, 18, 20, 18, 20, 21, 18, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302461237Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30247797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Max found gen is 22","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302491474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":Wait for a query_work_queue command to finish before sending IO "downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Generation requested: 22 >= found:22","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302511196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Next flush: 36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30252459Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302537934Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"":"downstairs","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302553907Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302566592Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"0","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302580326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"1","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30259404Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"client":"2","":"downstairs","session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302611952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302625326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"16ba463d-e310-421a-8b30-d007be717fa5 is now active with session: b17e8f9b-90f7-415e-837d-451267493c5c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.30264019Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302652734Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288,"session_id":"b17e8f9b-90f7-415e-837d-451267493c5c"} {"msg":"The guest has finished waiting for activation with:22","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302665909Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} {"msg":"Activated sub_volume 16ba463d-e310-421a-8b30-d007be717fa5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:44.302677583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 282 len 4096 data: 3 02/30 Write block 257 len 36864 data: 5 6 7 8 8 7 4 4 4 03/30 Read block 393 len 32768 04/30 Read block 272 len 40960 05/30 Read block 536 len 24576 06/30 Write block 412 len 28672 data: 3 4 4 3 2 3 3 07/30 Read block 313 len 36864 08/30 Read block 300 len 24576 09/30 Write block 409 len 16384 data: 4 3 3 4 10/30 Flush 11/30 Write block 84 len 36864 data: 2 3 3 2 2 2 3 3 3 12/30 Read block 515 len 16384 13/30 Write block 529 len 24576 data: 2 2 2 2 2 2 14/30 Read block 333 len 16384 15/30 Read block 160 len 36864 16/30 Write block 273 len 32768 data: 4 3 3 4 4 3 3 2 17/30 Flush 18/30 Write block 82 len 4096 data: 3 19/30 Read block 320 len 4096 20/30 Flush 21/30 Write block 125 len 4096 data: 6 22/30 Read block 207 len 8192 23/30 Flush 24/30 Write block 57 len 28672 data: 4 4 3 3 2 2 2 25/30 Read block 428 len 24576 26/30 Flush 27/30 Write block 55 len 40960 data: 4 5 5 5 4 4 3 3 3 3 28/30 Read block 411 len 28672 29/30 Read block 174 len 16384 30/30 Read block 482 len 24576 ---------------------------------------------------------------- Crucible gen:22 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Read 10 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Read 9 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 4 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Write 6 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 9 Done Sent Done false 1015 Acked Write 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 1 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 6 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 4 Done Sent Done false 1029 Acked Read 6 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 0 28 Done 29 3 30 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:44.315868467Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1288} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 22 20 22 40 33 40 T F T <--- 3 060-079 22 18 22 40 30 40 T F T <--- 4 080-099 22 20 22 38 33 38 F F F <--- 5 100-119 18 18 18 31 31 31 F F F 6 120-139 22 20 22 39 33 39 F F F <--- 7 140-159 21 21 21 35 35 35 F F F 8 160-179 18 18 18 30 30 30 F F F 9 180-199 20 20 20 33 33 33 F F F 10 200-219 20 20 20 33 33 33 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 22 18 22 36 30 36 F T F <--- 13 260-279 22 18 22 37 30 37 F T F <--- 14 280-299 22 20 22 37 34 37 F T F <--- 15 300-319 20 20 20 34 34 34 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 20 20 20 34 34 34 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 19 22 36 32 36 F F F <--- 21 420-439 21 21 21 35 35 35 F F F 22 440-459 21 21 21 35 35 35 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 21 21 21 35 35 35 F F F 26 520-539 22 12 22 37 20 37 F F F <--- 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 22, Max flush: 40 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 11 Verifying data now No change after verify Loop: 11 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 23 23 23 41 41 41 F F F 4 080-099 22 22 22 38 38 38 F F F 5 100-119 18 18 18 31 31 31 F F F 6 120-139 22 22 22 39 39 39 F F F 7 140-159 21 21 21 35 35 35 F F F 8 160-179 18 18 18 30 30 30 F F F 9 180-199 20 20 20 33 33 33 F F F 10 200-219 20 20 20 33 33 33 F F F 11 220-239 20 20 20 34 34 34 F F F 12 240-259 22 22 22 36 36 36 F F F 13 260-279 22 22 22 37 37 37 F F F 14 280-299 22 22 22 37 37 37 F F F 15 300-319 20 20 20 34 34 34 F F F 16 320-339 20 20 20 33 33 33 F F F 17 340-359 20 20 20 34 34 34 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 21 21 21 35 35 35 F F F 22 440-459 21 21 21 35 35 35 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 21 21 21 35 35 35 F F F 25 500-519 21 21 21 35 35 35 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 23, Max flush: 41 Begin loop 12 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.622295623Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.634169647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63453054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.634569603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.634582937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63460047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} {"msg":"Crucible 19295078-7460-4500-9575-93793e58ce5c has session id: 2a0d5e99-1817-4067-ade0-b106d0b9570c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63494669Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Upstairs opts: Upstairs UUID: 19295078-7460-4500-9575-93793e58ce5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635053643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Crucible stats registered with UUID: 19295078-7460-4500-9575-93793e58ce5c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635080842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:48.635094686Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635264542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635300476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63531597Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635410529Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"task":"crutest"} {"msg":"The guest has requested activation with gen:24","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635427491Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} {"msg":"19295078-7460-4500-9575-93793e58ce5c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635442665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635455499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"0","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635469673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"1","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635483767Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"2","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635497211Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635511555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635525789Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"ds_connection connected from Ok(127.0.0.1:51495)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635555606Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"0","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37411)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635570739Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"2","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37423)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.635585633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"io task","client":"1","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636368983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"0","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636407407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"1","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636424079Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"2","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636737754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636782454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[0]R flush_numbers[0..12]: [35, 20, 41, 41, 38, 31, 39, 35, 30, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636816719Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[0]R generation[0..12]: [21, 12, 23, 23, 22, 18, 22, 21, 18, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636833502Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636850245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[1]R flush_numbers[0..12]: [35, 20, 41, 41, 38, 31, 39, 35, 30, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636865978Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[1]R generation[0..12]: [21, 12, 23, 23, 22, 18, 22, 21, 18, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636881251Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636897704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[2]R flush_numbers[0..12]: [35, 20, 41, 41, 38, 31, 39, 35, 30, 33, 33, 34]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636915886Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[2]R generation[0..12]: [21, 12, 23, 23, 22, 18, 22, 21, 18, 20, 20, 20]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63693124Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636947423Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Max found gen is 24","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636961177Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Generation requested: 24 >= found:24","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.636981418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Next flush: 42","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637005408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637019332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"":"downstairs","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637033515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63704598Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"0","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637060094Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"1","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637076667Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"client":"2","":"downstairs","session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.63709122Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637104615Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"19295078-7460-4500-9575-93793e58ce5c is now active with session: 2a0d5e99-1817-4067-ade0-b106d0b9570c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637119728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637132622Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296,"session_id":"2a0d5e99-1817-4067-ade0-b106d0b9570c"} {"msg":"The guest has finished waiting for activation with:24","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637145867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} {"msg":"Activated sub_volume 19295078-7460-4500-9575-93793e58ce5c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:48.637158921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 299 len 12288 data: 3 3 4 02/30 Read block 267 len 20480 03/30 Flush 04/30 Write block 254 len 4096 data: 3 05/30 Read block 327 len 24576 06/30 Flush 07/30 Write block 92 len 40960 data: 4 4 4 5 4 3 2 2 2 2 08/30 Read block 189 len 16384 09/30 Read block 91 len 16384 10/30 Read block 400 len 16384 11/30 Write block 83 len 16384 data: 3 3 4 4 12/30 Write block 273 len 4096 data: 5 13/30 Write block 232 len 8192 data: 4 4 14/30 Write block 86 len 28672 data: 5 3 3 3 4 4 5 15/30 Read block 196 len 24576 16/30 Read block 180 len 32768 17/30 Write block 337 len 8192 data: 4 4 18/30 Flush 19/30 Flush 20/30 Write block 436 len 12288 data: 3 4 4 21/30 Write block 62 len 28672 data: 4 4 4 3 3 3 3 22/30 Write block 196 len 40960 data: 5 6 4 4 3 2 2 2 3 3 23/30 Write block 169 len 16384 data: 4 4 5 4 24/30 Read block 441 len 24576 25/30 Flush 26/30 Write block 106 len 36864 data: 3 3 3 3 3 3 3 3 3 27/30 Write block 132 len 16384 data: 3 3 3 4 28/30 Write block 497 len 32768 data: 5 4 4 4 3 3 3 3 29/30 Read block 518 len 40960 30/30 Read block 353 len 8192 ---------------------------------------------------------------- Crucible gen:24 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 5 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 4 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Write 4 Done Sent Done false 1011 Acked Write 1 Done Sent Done false 1012 Acked Write 2 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Write 10 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Read 6 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 8 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 2 Sent Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 0 30 Done 29 1 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:48.648470978Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1296} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 24 23 24 46 41 46 F F F <--- 4 080-099 24 22 24 44 38 44 F F F <--- 5 100-119 24 18 24 44 31 44 T F T <--- 6 120-139 22 22 22 39 39 39 T F T <--- 7 140-159 21 21 21 35 35 35 F F F 8 160-179 24 18 24 46 30 46 F F F <--- 9 180-199 24 20 24 46 33 46 F F F <--- 10 200-219 24 20 24 46 33 46 F F F <--- 11 220-239 24 20 24 44 34 44 F F F <--- 12 240-259 24 22 24 43 36 43 F F F <--- 13 260-279 24 22 24 44 37 44 F F F <--- 14 280-299 24 22 24 42 37 42 F T F <--- 15 300-319 24 20 24 42 34 42 F T F <--- 16 320-339 24 20 24 44 33 44 F F F <--- 17 340-359 20 20 20 34 34 34 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 24 21 24 46 35 46 F F F <--- 22 440-459 21 21 21 35 35 35 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 21 21 21 35 35 35 T F T <--- 25 500-519 21 21 21 35 35 35 T F T <--- 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 24, Max flush: 46 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 12 Verifying data now No change after verify Loop: 12 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 21 21 21 35 35 35 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 23 23 23 41 41 41 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 24 24 24 44 44 44 F F F 5 100-119 25 25 25 47 47 47 F F F 6 120-139 25 25 25 47 47 47 F F F 7 140-159 21 21 21 35 35 35 F F F 8 160-179 24 24 24 46 46 46 F F F 9 180-199 24 24 24 46 46 46 F F F 10 200-219 24 24 24 46 46 46 F F F 11 220-239 24 24 24 44 44 44 F F F 12 240-259 24 24 24 43 43 43 F F F 13 260-279 24 24 24 44 44 44 F F F 14 280-299 24 24 24 42 42 42 F F F 15 300-319 24 24 24 42 42 42 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 20 20 20 34 34 34 F F F 18 360-379 16 16 16 28 28 28 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 24 24 24 46 46 46 F F F 22 440-459 21 21 21 35 35 35 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 25 25 25 47 47 47 F F F 25 500-519 25 25 25 47 47 47 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 14 14 14 23 23 23 F F F 29 580-599 18 18 18 31 31 31 F F F Max gen: 25, Max flush: 47 Begin loop 13 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.972867747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98453622Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.984840398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.984878512Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.984890576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.984908279Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} {"msg":"Crucible e6fdd8ed-dc97-4f2f-9131-6c56073746b3 has session id: dd20c05a-519f-411e-ba2c-c5215f885963","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98525096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Upstairs opts: Upstairs UUID: e6fdd8ed-dc97-4f2f-9131-6c56073746b3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985287664Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Crucible stats registered with UUID: e6fdd8ed-dc97-4f2f-9131-6c56073746b3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985302608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:52.985316042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98548145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98550624Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985520983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985695608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"task":"crutest"} {"msg":"The guest has requested activation with gen:26","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98571263Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} {"msg":"e6fdd8ed-dc97-4f2f-9131-6c56073746b3 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985729983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985742977Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"0","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985756781Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"1","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985770755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"2","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985784239Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98580462Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985819084Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"ds_connection connected from Ok(127.0.0.1:64893)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985834148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"0","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"ds_connection connected from Ok(127.0.0.1:58096)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985848791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"2","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"ds_connection connected from Ok(127.0.0.1:61647)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.985961043Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"io task","client":"1","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.986614099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"1","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.986643117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"0","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.986667616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"2","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.986977472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987010788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[0]R flush_numbers[0..12]: [35, 20, 41, 46, 44, 47, 47, 35, 46, 46, 46, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.98707345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[0]R generation[0..12]: [21, 12, 23, 24, 24, 25, 25, 21, 24, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987090453Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987106836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[1]R flush_numbers[0..12]: [35, 20, 41, 46, 44, 47, 47, 35, 46, 46, 46, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987122379Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[1]R generation[0..12]: [21, 12, 23, 24, 24, 25, 25, 21, 24, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987137283Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987153526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[2]R flush_numbers[0..12]: [35, 20, 41, 46, 44, 47, 47, 35, 46, 46, 46, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987169169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[2]R generation[0..12]: [21, 12, 23, 24, 24, 25, 25, 21, 24, 24, 24, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987184632Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987200365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Max found gen is 26","Wait for a query_work_queue command to finish before sending IO v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987219627Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Generation requested: 26 >= found:26","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987233981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Next flush: 48","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987247285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987260549Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"":"downstairs","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987274483Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987288947Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"0","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.9873057Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"1","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987320763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"client":"2","":"downstairs","session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987335197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987348811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"e6fdd8ed-dc97-4f2f-9131-6c56073746b3 is now active with session: dd20c05a-519f-411e-ba2c-c5215f885963","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987364034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987376649Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304,"session_id":"dd20c05a-519f-411e-ba2c-c5215f885963"} {"msg":"The guest has finished waiting for activation with:26","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987389903Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} {"msg":"Activated sub_volume e6fdd8ed-dc97-4f2f-9131-6c56073746b3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:52.987401758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 317 len 12288 data: 3 2 2 02/30 Flush 03/30 Flush 04/30 Flush 05/30 Write block 576 len 32768 data: 2 2 3 4 4 4 3 2 06/30 Flush 07/30 Write block 195 len 24576 data: 4 6 7 5 5 4 08/30 Write block 429 len 24576 data: 3 3 3 3 3 3 09/30 Read block 521 len 36864 10/30 Write block 349 len 28672 data: 5 5 4 5 4 4 4 11/30 Read block 412 len 12288 12/30 Read block 208 len 40960 13/30 Write block 49 len 28672 data: 5 5 4 4 4 4 5 14/30 Write block 356 len 8192 data: 3 4 15/30 Read block 145 len 40960 16/30 Read block 264 len 32768 17/30 Write block 376 len 8192 data: 2 2 18/30 Write block 439 len 40960 data: 4 4 4 3 3 3 3 3 3 3 19/30 Read block 98 len 28672 20/30 Read block 407 len 40960 21/30 Write block 11 len 32768 data: 3 2 2 3 4 4 4 4 22/30 Read block 262 len 40960 23/30 Write block 98 len 32768 data: 3 3 3 3 2 2 3 2 24/30 Flush 25/30 Write block 16 len 4096 data: 5 26/30 Read block 395 len 32768 27/30 Write block 148 len 36864 data: 4 4 5 5 3 3 3 3 3 28/30 Write block 509 len 16384 data: 2 2 2 2 29/30 Read block 41 len 4096 30/30 Write block 201 len 36864 data: 3 3 3 4 4 3 2 2 3 ---------------------------------------------------------------- Crucible gen:26 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 6 Done Done Done false 1007 Acked Write 6 Done Done Done false 1008 Acked Read 9 Done Done Done false 1009 Acked Write 7 Done Done Done false 1010 Acked Read 3 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Read 10 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Write 2 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Write 8 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 8 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 18 20 Done 23 23 6 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:53.000861175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1304} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 26 26 21 52 52 35 T T F <--- 1 020-039 12 12 12 20 20 20 F F F 2 040-059 26 26 23 52 52 41 F F F <--- 3 060-079 24 24 24 46 46 46 F F F 4 080-099 26 26 24 52 52 44 F F F <--- 5 100-119 26 26 25 52 52 47 F F F <--- 6 120-139 25 25 25 47 47 47 F F F 7 140-159 21 21 21 35 35 35 T T F <--- 8 160-179 24 24 24 46 46 46 F F F 9 180-199 26 26 24 52 52 46 F F T <--- 10 200-219 26 26 24 52 52 46 T T T <--- 11 220-239 24 24 24 44 44 44 F F F 12 240-259 24 24 24 43 43 43 F F F 13 260-279 24 24 24 44 44 44 F F F 14 280-299 24 24 24 42 42 42 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 26 26 20 52 52 34 F F T <--- 18 360-379 26 26 16 52 52 28 F F F <--- 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 26 26 24 52 52 46 F F T <--- 22 440-459 26 26 21 52 52 35 F F F <--- 23 460-479 15 15 15 25 25 25 F F F 24 480-499 25 25 25 47 47 47 F F F 25 500-519 25 25 25 47 47 47 T T F <--- 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 26 26 26 51 51 51 F F F 29 580-599 26 26 26 51 51 51 F F F Max gen: 26, Max flush: 52 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 13 Verifying data now No change after verify Loop: 13 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 26 26 26 52 52 52 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 26 26 26 52 52 52 F F F 5 100-119 26 26 26 52 52 52 F F F 6 120-139 25 25 25 47 47 47 F F F 7 140-159 27 27 27 53 53 53 F F F 8 160-179 24 24 24 46 46 46 F F F 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 24 24 24 44 44 44 F F F 12 240-259 24 24 24 43 43 43 F F F 13 260-279 24 24 24 44 44 44 F F F 14 280-299 24 24 24 42 42 42 F F F 15 300-319 26 26 26 48 48 48 F F F 16 320-339 24 24 24 44 44 44 F F F 17 340-359 26 26 26 52 52 52 F F F 18 360-379 26 26 26 52 52 52 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 26 26 26 52 52 52 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 25 25 25 47 47 47 F F F 25 500-519 27 27 27 53 53 53 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 26 26 26 51 51 51 F F F 29 580-599 26 26 26 51 51 51 F F F Max gen: 27, Max flush: 53 Begin loop 14 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.317943204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:57.328600855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.328650014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339110551Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339402204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339440947Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339453202Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339471084Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} {"msg":"Crucible f1ae9138-5ff4-46a1-8b20-2ee1b8de3e89 has session id: e40e6ee6-7bb3-49e5-96f4-28339c8b9d95","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339814096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Upstairs opts: Upstairs UUID: f1ae9138-5ff4-46a1-8b20-2ee1b8de3e89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339852479Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Crucible stats registered with UUID: f1ae9138-5ff4-46a1-8b20-2ee1b8de3e89","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.339868412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:57.339881726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340039078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340065117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.34007992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340221109Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"task":"crutest"} {"msg":"The guest has requested activation with gen:28","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340240141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} {"msg":"f1ae9138-5ff4-46a1-8b20-2ee1b8de3e89 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340261102Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340273736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"0","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.34028785Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"1","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340301794Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"2","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340315368Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340329692Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340343796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"ds_connection connected from Ok(127.0.0.1:58457)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340383899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"0","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"ds_connection connected from Ok(127.0.0.1:62986)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340531135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"2","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"ds_connection connected from Ok(127.0.0.1:64220)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.340558473Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"io task","client":"1","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341036626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"1","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341065533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"2","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341223515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"0","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341407485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341449757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[0]R flush_numbers[0..12]: [53, 20, 52, 46, 52, 52, 47, 53, 46, 52, 53, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341584009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[0]R generation[0..12]: [27, 12, 26, 24, 26, 26, 25, 27, 24, 26, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341600581Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341617194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[1]R flush_numbers[0..12]: [53, 20, 52, 46, 52, 52, 47, 53, 46, 52, 53, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341632677Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[1]R generation[0..12]: [27, 12, 26, 24, 26, 26, 25, 27, 24, 26, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341647991Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341664544Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[2]R flush_numbers[0..12]: [53, 20, 52, 46, 52, 52, 47, 53, 46, 52, 53, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341682656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[2]R generation[0..12]: [27, 12, 26, 24, 26, 26, 25, 27, 24, 26, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341705146Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341721749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Max found gen is 28","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:49:57.341735503Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Generation requested: 28 >= found:28","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341754765Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Next flush: 54","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341767779Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341781083Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"":"downstairs","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341794897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341807712Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"0","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341821925Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"1","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341838708Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"client":"2","":"downstairs","session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341853492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341866956Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"f1ae9138-5ff4-46a1-8b20-2ee1b8de3e89 is now active with session: e40e6ee6-7bb3-49e5-96f4-28339c8b9d95","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.34188163Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341894464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312,"session_id":"e40e6ee6-7bb3-49e5-96f4-28339c8b9d95"} {"msg":"The guest has finished waiting for activation with:28","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341909617Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} {"msg":"Activated sub_volume f1ae9138-5ff4-46a1-8b20-2ee1b8de3e89","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:49:57.341921372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 578 len 32768 data: 4 5 5 5 4 3 3 3 02/30 Read block 124 len 20480 03/30 Read block 364 len 16384 04/30 Write block 51 len 20480 data: 5 5 5 5 6 05/30 Read block 29 len 36864 06/30 Read block 36 len 20480 07/30 Flush 08/30 Flush 09/30 Write block 341 len 12288 data: 4 5 5 10/30 Write block 170 len 40960 data: 5 6 5 4 4 4 3 2 2 2 11/30 Read block 235 len 16384 12/30 Flush 13/30 Write block 319 len 24576 data: 3 2 2 2 2 2 14/30 Read block 2 len 8192 15/30 Read block 6 len 4096 16/30 Read block 488 len 24576 17/30 Read block 419 len 4096 18/30 Flush 19/30 Write block 84 len 16384 data: 4 5 6 4 20/30 Read block 210 len 4096 21/30 Write block 297 len 20480 data: 5 3 4 4 5 22/30 Read block 190 len 36864 23/30 Read block 414 len 8192 24/30 Flush 25/30 Write block 285 len 32768 data: 4 7 7 6 6 4 4 3 26/30 Read block 446 len 4096 27/30 Read block 363 len 36864 28/30 Read block 398 len 36864 29/30 Read block 530 len 8192 30/30 Read block 204 len 28672 ---------------------------------------------------------------- Crucible gen:28 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Read 4 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Read 5 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 3 Sent Done Done false 1009 Acked Write 10 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 2 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 6 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 8 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Read 9 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Read 7 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:49:57.354746924Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1312} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 26 28 28 52 54 54 F F F <--- 3 060-079 24 24 24 46 46 46 F F F 4 080-099 26 28 28 52 58 58 F F F <--- 5 100-119 26 26 26 52 52 52 F F F 6 120-139 25 25 25 47 47 47 F F F 7 140-159 27 27 27 53 53 53 F F F 8 160-179 24 28 28 46 56 56 F F F <--- 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 24 24 24 44 44 44 F F F 12 240-259 24 24 24 43 43 43 F F F 13 260-279 24 24 24 44 44 44 F F F 14 280-299 24 28 28 42 58 58 F T T <--- 15 300-319 26 28 28 48 58 58 F F F <--- 16 320-339 24 28 28 44 57 57 F F F <--- 17 340-359 26 28 28 52 56 56 F F F <--- 18 360-379 26 26 26 52 52 52 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 26 26 26 52 52 52 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 25 25 25 47 47 47 F F F 25 500-519 27 27 27 53 53 53 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 26 28 28 51 54 54 F F F <--- 29 580-599 26 28 28 51 54 54 F F F <--- Max gen: 28, Max flush: 58 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 14 Verifying data now No change after verify Loop: 14 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 12 12 12 20 20 20 F F F 2 040-059 28 28 28 54 54 54 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 28 28 28 58 58 58 F F F 5 100-119 26 26 26 52 52 52 F F F 6 120-139 25 25 25 47 47 47 F F F 7 140-159 27 27 27 53 53 53 F F F 8 160-179 28 28 28 56 56 56 F F F 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 24 24 24 44 44 44 F F F 12 240-259 24 24 24 43 43 43 F F F 13 260-279 24 24 24 44 44 44 F F F 14 280-299 29 29 29 59 59 59 F F F 15 300-319 28 28 28 58 58 58 F F F 16 320-339 28 28 28 57 57 57 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 26 26 26 52 52 52 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 22 22 22 36 36 36 F F F 21 420-439 26 26 26 52 52 52 F F F 22 440-459 26 26 26 52 52 52 F F F 23 460-479 15 15 15 25 25 25 F F F 24 480-499 25 25 25 47 47 47 F F F 25 500-519 27 27 27 53 53 53 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 12 12 12 20 20 20 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 28 28 28 54 54 54 F F F Max gen: 29, Max flush: 59 Begin loop 15 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.671754665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.683644072Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.683974419Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684023128Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684035992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684053854Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} {"msg":"Crucible 5913aaa8-93ed-4669-bff1-a5e0040adc5b has session id: 52d14ee1-eca5-4ebb-a659-0614fd6c1d06","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684379373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Upstairs opts: Upstairs UUID: 5913aaa8-93ed-4669-bff1-a5e0040adc5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684430901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Crucible stats registered with UUID: 5913aaa8-93ed-4669-bff1-a5e0040adc5b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684446974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:01.684460078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684666669Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684702373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684717867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684843482Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"task":"crutest"} {"msg":"The guest has requested activation with gen:30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684861434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} {"msg":"5913aaa8-93ed-4669-bff1-a5e0040adc5b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684881766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.68489448Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"0","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684908904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"1","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684922888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"2","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684936922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684951026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684965139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"ds_connection connected from Ok(127.0.0.1:35686)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.684984461Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"1","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"ds_connection connected from Ok(127.0.0.1:57339)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.685003863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"0","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"ds_connection connected from Ok(127.0.0.1:61956)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.685024244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"io task","client":"2","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.685711916Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"0","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.685740154Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"1","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.685756636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"2","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686126676Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.68614076Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[0]R flush_numbers[0..12]: [53, 20, 54, 46, 58, 52, 47, 53, 56, 52, 53, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686176704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[0]R generation[0..12]: [27, 12, 28, 24, 28, 26, 25, 27, 28, 26, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686193867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.68621047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[1]R flush_numbers[0..12]: [53, 20, 54, 46, 58, 52, 47, 53, 56, 52, 53, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686226223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[1]R generation[0..12]: [27, 12, 28, 24, 28, 26, 25, 27, 28, 26, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686241206Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686257529Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[2]R flush_numbers[0..12]: [53, 20, 54, 46, 58, 52, 47, 53, 56, 52, 53, 44]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686273143Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[2]R generation[0..12]: [27, 12, 28, 24, 28, 26, 25, 27, 28, 26, 27, 24]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686288496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686304989Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Max found gen is 30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686318733Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Generation requested: 30 >= found:30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686339024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Next flush: 60","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686352168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686365313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"":"downstairs","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686379496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686392141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"0","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686407904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"1","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686421828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"client":"2","":"downstairs","session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686437861Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686451555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"5913aaa8-93ed-4669-bff1-a5e0040adc5b is now active with session: 52d14ee1-eca5-4ebb-a659-0614fd6c1d06","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686466609Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686479403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320,"session_id":"52d14ee1-eca5-4ebb-a659-0614fd6c1d06"} {"msg":"The guest has finished waiting for activation with:30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686492537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} {"msg":"Activated sub_volume 5913aaa8-93ed-4669-bff1-a5e0040adc5b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:01.686504112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 12288 data: 4 5 5 02/30 Flush 03/30 Write block 92 len 36864 data: 6 5 5 6 5 4 4 4 4 04/30 Read block 272 len 16384 05/30 Flush 06/30 Write block 309 len 4096 data: 4 07/30 Read block 491 len 32768 08/30 Write block 125 len 8192 data: 7 4 09/30 Write block 435 len 32768 data: 4 4 5 5 5 5 5 4 10/30 Read block 182 len 40960 11/30 Write block 400 len 28672 data: 2 3 4 4 4 4 3 12/30 Write block 549 len 24576 data: 4 4 4 3 4 4 13/30 Write block 221 len 16384 data: 3 3 3 3 14/30 Write block 232 len 8192 data: 5 5 15/30 Write block 477 len 36864 data: 2 2 3 3 3 5 5 4 4 16/30 Read block 153 len 32768 17/30 Flush 18/30 Write block 480 len 20480 data: 4 4 6 6 5 19/30 Write block 265 len 20480 data: 5 4 3 2 3 20/30 Write block 508 len 28672 data: 3 3 3 3 3 3 3 21/30 Read block 7 len 36864 22/30 Read block 342 len 12288 23/30 Read block 222 len 40960 24/30 Read block 407 len 40960 25/30 Write block 326 len 16384 data: 2 3 3 3 26/30 Read block 9 len 20480 27/30 Write block 26 len 16384 data: 3 2 2 3 28/30 Read block 315 len 28672 29/30 Read block 420 len 40960 30/30 Read block 431 len 24576 ---------------------------------------------------------------- Crucible gen:30 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 1 Done Done Done false 1006 Acked Read 8 Done Done Done false 1007 Acked Write 2 Done Done Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Write 7 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Write 2 Done Sent Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 9 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Read 10 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Read 7 Done Sent Done false 1028 Acked Read 10 Done Sent Done false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 22 1 23 Done 25 3 24 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1004)) Some(JobId(1016)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:01.698872992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1320} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 12 12 12 20 20 20 T F T <--- 2 040-059 30 30 30 60 60 60 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 30 30 30 61 61 61 F F F 6 120-139 30 25 30 62 47 62 F T F <--- 7 140-159 27 27 27 53 53 53 F F F 8 160-179 28 28 28 56 56 56 F F F 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 30 24 30 62 44 62 F F F <--- 12 240-259 24 24 24 43 43 43 F F F 13 260-279 24 24 24 44 44 44 T F T <--- 14 280-299 29 29 29 59 59 59 F F F 15 300-319 30 28 30 62 58 62 F T F <--- 16 320-339 28 28 28 57 57 57 T F T <--- 17 340-359 28 28 28 56 56 56 F F F 18 360-379 26 26 26 52 52 52 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 30 22 30 62 36 62 F F F <--- 21 420-439 30 26 30 62 52 62 F F F <--- 22 440-459 30 26 30 62 52 62 F F F <--- 23 460-479 30 15 30 62 25 62 F F F <--- 24 480-499 30 25 30 62 47 62 T F T <--- 25 500-519 27 27 27 53 53 53 T F T <--- 26 520-539 22 22 22 37 37 37 F F F 27 540-559 30 12 30 62 20 62 F F F <--- 28 560-579 28 28 28 54 54 54 F F F 29 580-599 28 28 28 54 54 54 F F F Max gen: 30, Max flush: 62 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 15 Verifying data now No change after verify Loop: 15 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 31 31 31 63 63 63 F F F 2 040-059 30 30 30 60 60 60 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 30 30 30 61 61 61 F F F 6 120-139 30 30 30 62 62 62 F F F 7 140-159 27 27 27 53 53 53 F F F 8 160-179 28 28 28 56 56 56 F F F 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 30 30 30 62 62 62 F F F 12 240-259 24 24 24 43 43 43 F F F 13 260-279 31 31 31 63 63 63 F F F 14 280-299 29 29 29 59 59 59 F F F 15 300-319 30 30 30 62 62 62 F F F 16 320-339 31 31 31 63 63 63 F F F 17 340-359 28 28 28 56 56 56 F F F 18 360-379 26 26 26 52 52 52 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 30 30 30 62 62 62 F F F 21 420-439 30 30 30 62 62 62 F F F 22 440-459 30 30 30 62 62 62 F F F 23 460-479 30 30 30 62 62 62 F F F 24 480-499 31 31 31 63 63 63 F F F 25 500-519 31 31 31 63 63 63 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 28 28 28 54 54 54 F F F Max gen: 31, Max flush: 63 Begin loop 16 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.020528678Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:06.031274101Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.0313227Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.041736757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042049681Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042089534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042101908Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042119831Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} {"msg":"Crucible 2cb38458-39ab-41bc-878e-3b8dc848b6d2 has session id: 41bef0f4-e25c-4658-9170-eabddb837f52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042482054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Upstairs opts: Upstairs UUID: 2cb38458-39ab-41bc-878e-3b8dc848b6d2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042519348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Crucible stats registered with UUID: 2cb38458-39ab-41bc-878e-3b8dc848b6d2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042535251Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:06.042548625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042748458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042785402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042800945Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042893945Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"task":"crutest"} {"msg":"The guest has requested activation with gen:32","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042911737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} {"msg":"2cb38458-39ab-41bc-878e-3b8dc848b6d2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.04292864Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042941954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"0","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042956148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"1","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042970392Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"2","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.042984356Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.0429988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043030856Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:61793)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043048858Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"2","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:36500)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043064021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"1","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"ds_connection connected from Ok(127.0.0.1:56231)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043125915Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"io task","client":"0","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043853569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"1","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043883456Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"0","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.043900239Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"2","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044260953Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044276926Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[0]R flush_numbers[0..12]: [53, 63, 60, 46, 61, 61, 62, 53, 56, 52, 53, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044331372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[0]R generation[0..12]: [27, 31, 30, 24, 30, 30, 30, 27, 28, 26, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044348165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044364388Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[1]R flush_numbers[0..12]: [53, 63, 60, 46, 61, 61, 62, 53, 56, 52, 53, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044379921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[1]R generation[0..12]: [27, 31, 30, 24, 30, 30, 30, 27, 28, 26, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044395524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044411767Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[2]R flush_numbers[0..12]: [53, 63, 60, 46, 61, 61, 62, 53, 56, 52, 53, 62]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044430459Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[2]R generation[0..12]: [27, 31, 30, 24, 30, 30, 30, 27, 28, 26, 27, 30]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044445963Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044462345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Max found gen is 32","v":0,"name":"crucible","level":30,"Wait for a query_work_queue command to finish before sending IO time":"2026-03-12T22:50:06.0444759Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Generation requested: 32 >= found:32","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044496621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Next flush: 64","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044510205Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044523569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"":"downstairs","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044537853Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044550677Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"0","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044564941Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"1","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044579025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"client":"2","":"downstairs","session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044593329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044606603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"2cb38458-39ab-41bc-878e-3b8dc848b6d2 is now active with session: 41bef0f4-e25c-4658-9170-eabddb837f52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044625585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044638449Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328,"session_id":"41bef0f4-e25c-4658-9170-eabddb837f52"} {"msg":"The guest has finished waiting for activation with:32","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044651843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} {"msg":"Activated sub_volume 2cb38458-39ab-41bc-878e-3b8dc848b6d2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:06.044665038Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 266 len 28672 data: 5 4 3 4 3 3 4 02/30 Read block 484 len 16384 03/30 Write block 358 len 8192 data: 4 4 04/30 Write block 127 len 20480 data: 3 3 3 2 2 05/30 Write block 112 len 20480 data: 4 4 4 3 3 06/30 Write block 51 len 16384 data: 6 6 6 6 07/30 Read block 325 len 40960 08/30 Read block 226 len 16384 09/30 Read block 179 len 40960 10/30 Write block 496 len 28672 data: 4 6 5 5 5 4 4 11/30 Write block 36 len 20480 data: 4 2 2 2 3 12/30 Read block 144 len 28672 13/30 Read block 281 len 16384 14/30 Write block 416 len 32768 data: 3 4 4 3 3 3 4 4 15/30 Write block 23 len 12288 data: 2 3 3 16/30 Read block 517 len 40960 17/30 Write block 236 len 24576 data: 4 4 5 6 6 6 18/30 Read block 493 len 20480 19/30 Read block 479 len 40960 20/30 Write block 486 len 16384 data: 4 4 5 5 21/30 Flush 22/30 Write block 585 len 8192 data: 4 4 23/30 Flush 24/30 Write block 356 len 12288 data: 4 5 5 25/30 Read block 145 len 32768 26/30 Write block 133 len 32768 data: 4 4 5 4 4 4 4 4 27/30 Read block 202 len 32768 28/30 Write block 167 len 24576 data: 3 4 5 6 7 6 29/30 Read block 186 len 36864 30/30 Write block 378 len 4096 data: 3 ---------------------------------------------------------------- Crucible gen:32 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 4 Done Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Write 5 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 7 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 4 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Read 10 Sent Done Done false 1016 Acked Write 6 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 10 Sent Done Done false 1019 Acked Write 4 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Read 8 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 8 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 9 Sent Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 2 31 Done 2 29 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1022)) Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:06.057499095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1328} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 31 32 32 63 64 64 F F F <--- 2 040-059 30 32 32 60 64 64 F F F <--- 3 060-079 24 24 24 46 46 46 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 30 32 32 61 64 64 F F F <--- 6 120-139 30 32 32 62 64 64 F T T <--- 7 140-159 27 27 27 53 53 53 F T T <--- 8 160-179 28 28 28 56 56 56 F T T <--- 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 30 32 32 62 64 64 F F F <--- 12 240-259 24 32 32 43 64 64 F F F <--- 13 260-279 31 32 32 63 64 64 T F F <--- 14 280-299 29 29 29 59 59 59 F F F 15 300-319 30 30 30 62 62 62 F F F 16 320-339 31 31 31 63 63 63 F F F 17 340-359 28 32 32 56 64 64 F T T <--- 18 360-379 26 26 26 52 52 52 F T T <--- 19 380-399 21 21 21 35 35 35 F F F 20 400-419 30 32 32 62 64 64 F F F <--- 21 420-439 30 32 32 62 64 64 F F F <--- 22 440-459 30 30 30 62 62 62 F F F 23 460-479 30 30 30 62 62 62 F F F 24 480-499 31 32 32 63 64 64 F F F <--- 25 500-519 31 32 32 63 64 64 F F F <--- 26 520-539 22 22 22 37 37 37 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 28 32 32 54 65 65 F F F <--- Max gen: 32, Max flush: 65 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 16 Verifying data now No change after verify Loop: 16 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 27 27 27 53 53 53 F F F 1 020-039 32 32 32 64 64 64 F F F 2 040-059 32 32 32 64 64 64 F F F 3 060-079 24 24 24 46 46 46 F F F 4 080-099 30 30 30 61 61 61 F F F 5 100-119 32 32 32 64 64 64 F F F 6 120-139 33 33 33 66 66 66 F F F 7 140-159 33 33 33 66 66 66 F F F 8 160-179 33 33 33 66 66 66 F F F 9 180-199 26 26 26 52 52 52 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 32 32 32 64 64 64 F F F 12 240-259 32 32 32 64 64 64 F F F 13 260-279 32 32 32 64 64 64 F F F 14 280-299 29 29 29 59 59 59 F F F 15 300-319 30 30 30 62 62 62 F F F 16 320-339 31 31 31 63 63 63 F F F 17 340-359 33 33 33 66 66 66 F F F 18 360-379 33 33 33 66 66 66 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 32 32 32 64 64 64 F F F 22 440-459 30 30 30 62 62 62 F F F 23 460-479 30 30 30 62 62 62 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 22 22 22 37 37 37 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 33, Max flush: 66 Begin loop 17 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.396495126Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.408355876Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.408674348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.408714041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.408726595Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.408744458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} {"msg":"Crucible c9d197c1-5820-42de-a3df-bf7b502ea851 has session id: 21351ab1-6940-4eb0-88b2-88c4bff26175","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409100153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Upstairs opts: Upstairs UUID: c9d197c1-5820-42de-a3df-bf7b502ea851, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409152391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Crucible stats registered with UUID: c9d197c1-5820-42de-a3df-bf7b502ea851","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409168434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:10.409181758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409364039Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409400913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409416526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409516053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"task":"crutest"} {"msg":"The guest has requested activation with gen:34","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409532246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} {"msg":"c9d197c1-5820-42de-a3df-bf7b502ea851 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409550258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409563173Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"0","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409577167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"1","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409591521Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"2","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409605664Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409649895Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409672945Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"ds_connection connected from Ok(127.0.0.1:57131)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409688798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"0","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"ds_connection connected from Ok(127.0.0.1:64296)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409703972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"1","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"ds_connection connected from Ok(127.0.0.1:42965)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.409719025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"io task","client":"2","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410472009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"1","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410523667Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"2","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410542129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"0","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410818899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410864169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[0]R flush_numbers[0..12]: [53, 64, 64, 46, 61, 64, 66, 66, 66, 52, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.41095403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[0]R generation[0..12]: [27, 32, 32, 24, 30, 32, 33, 33, 33, 26, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410971343Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.410987875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[1]R flush_numbers[0..12]: [53, 64, 64, 46, 61, 64, 66, 66, 66, 52, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411003529Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[1]R generation[0..12]: [27, 32, 32, 24, 30, 32, 33, 33, 33, 26, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411018992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411035155Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[2]R flush_numbers[0..12]: [53, 64, 64, 46, 61, 64, 66, 66, 66, 52, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411050598Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[2]R generation[0..12]: [27, 32, 32, 24, 30, 32, 33, 33, 33, 26, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411065802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411082634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Max found gen is 34","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-03-12T22:50:10.411102056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Generation requested: 34 >= found:34","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.41111637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Next flush: 67","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411129684Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411143088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"":"downstairs","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411157092Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411171766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"0","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:50:10.4111859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"1","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411202812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"client":"2","":"downstairs","session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411217366Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.41123071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"c9d197c1-5820-42de-a3df-bf7b502ea851 is now active with session: 21351ab1-6940-4eb0-88b2-88c4bff26175","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411245434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411257958Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336,"session_id":"21351ab1-6940-4eb0-88b2-88c4bff26175"} {"msg":"The guest has finished waiting for activation with:34","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411271323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} {"msg":"Activated sub_volume c9d197c1-5820-42de-a3df-bf7b502ea851","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:10.411283027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 20480 data: 3 2 2 2 2 02/30 Write block 284 len 40960 data: 4 5 8 8 7 7 5 5 4 3 03/30 Write block 454 len 4096 data: 3 04/30 Write block 338 len 36864 data: 5 3 3 5 6 6 5 4 3 05/30 Write block 78 len 32768 data: 3 3 3 3 4 4 5 6 06/30 Read block 329 len 28672 07/30 Read block 573 len 24576 08/30 Write block 466 len 16384 data: 4 6 6 6 09/30 Write block 8 len 16384 data: 4 4 4 4 10/30 Flush 11/30 Write block 284 len 36864 data: 5 6 9 9 8 8 6 6 5 12/30 Read block 434 len 40960 13/30 Flush 14/30 Write block 376 len 4096 data: 3 15/30 Read block 307 len 8192 16/30 Write block 525 len 8192 data: 3 2 17/30 Read block 554 len 32768 18/30 Read block 425 len 4096 19/30 Read block 151 len 32768 20/30 Read block 268 len 4096 21/30 Write block 22 len 36864 data: 2 3 4 4 4 3 3 4 4 22/30 Write block 28 len 36864 data: 4 5 5 4 4 4 4 4 5 23/30 Write block 531 len 12288 data: 3 3 3 24/30 Read block 408 len 36864 25/30 Read block 464 len 28672 26/30 Write block 424 len 4096 data: 4 27/30 Read block 265 len 36864 28/30 Write block 93 len 20480 data: 6 6 7 6 5 29/30 Read block 595 len 4096 30/30 Read block 385 len 8192 ---------------------------------------------------------------- Crucible gen:34 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 1 Done Done Done false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 8 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Read 6 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 8 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:10.422412694Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1336} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 27 67 67 53 F F F <--- 1 020-039 32 32 32 64 64 64 T T F <--- 2 040-059 32 32 32 64 64 64 F F F 3 060-079 34 34 24 67 67 46 F F F <--- 4 080-099 34 34 30 67 67 61 T T F <--- 5 100-119 32 32 32 64 64 64 F F F 6 120-139 33 33 33 66 66 66 F F F 7 140-159 33 33 33 66 66 66 F F F 8 160-179 34 34 33 67 67 66 F F T <--- 9 180-199 34 34 26 67 67 52 F F T <--- 10 200-219 27 27 27 53 53 53 F F F 11 220-239 32 32 32 64 64 64 F F F 12 240-259 32 32 32 64 64 64 F F F 13 260-279 32 32 32 64 64 64 F F F 14 280-299 34 34 29 68 68 59 F F T <--- 15 300-319 30 30 30 62 62 62 F F F 16 320-339 34 34 31 67 67 63 F F F <--- 17 340-359 34 34 33 67 67 66 F F F <--- 18 360-379 33 33 33 66 66 66 T T F <--- 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 32 32 32 64 64 64 T T F <--- 22 440-459 34 34 30 67 67 62 F F T <--- 23 460-479 34 34 30 67 67 62 F F F <--- 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 22 22 22 37 37 37 T T F <--- 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 34, Max flush: 68 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 17 Verifying data now No change after verify Loop: 17 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 67 67 67 F F F 1 020-039 35 35 35 69 69 69 F F F 2 040-059 32 32 32 64 64 64 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 69 69 69 F F F 5 100-119 32 32 32 64 64 64 F F F 6 120-139 33 33 33 66 66 66 F F F 7 140-159 33 33 33 66 66 66 F F F 8 160-179 34 34 34 67 67 67 F F F 9 180-199 34 34 34 67 67 67 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 32 32 32 64 64 64 F F F 12 240-259 32 32 32 64 64 64 F F F 13 260-279 32 32 32 64 64 64 F F F 14 280-299 34 34 34 68 68 68 F F F 15 300-319 30 30 30 62 62 62 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 34 34 34 67 67 67 F F F 18 360-379 35 35 35 69 69 69 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 35 35 35 69 69 69 F F F 22 440-459 34 34 34 67 67 67 F F F 23 460-479 34 34 34 67 67 67 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 35, Max flush: 69 Begin loop 18 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.745299267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.756998796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.757320177Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.757361209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.757373284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.757390846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} {"msg":"Crucible 324d3363-e93b-425c-aaaa-c8cbc1711ec7 has session id: b2cce334-6bbd-4858-ad7e-43825338be04","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.75772902Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Upstairs opts: Upstairs UUID: 324d3363-e93b-425c-aaaa-c8cbc1711ec7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.757768093Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Crucible stats registered with UUID: 324d3363-e93b-425c-aaaa-c8cbc1711ec7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.757784176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:14.75779747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758021803Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758061636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758076719Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758201175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"task":"crutest"} {"msg":"The guest has requested activation with gen:36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758219247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} {"msg":"324d3363-e93b-425c-aaaa-c8cbc1711ec7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.75823618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758248875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"0","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758262879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"1","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758276803Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"2","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758290437Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.75830455Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758319034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"ds_connection connected from Ok(127.0.0.1:52074)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758464591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"1","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"ds_connection connected from Ok(127.0.0.1:42699)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758584939Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"2","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"ds_connection connected from Ok(127.0.0.1:44954)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.758612997Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"io task","client":"0","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.75915977Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"1","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759187628Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"0","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759314153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"2","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759541574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759579938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[0]R flush_numbers[0..12]: [67, 69, 64, 67, 69, 64, 66, 66, 67, 67, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759688901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[0]R generation[0..12]: [34, 35, 32, 34, 35, 32, 33, 33, 34, 34, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759705863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759722196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[1]R flush_numbers[0..12]: [67, 69, 64, 67, 69, 64, 66, 66, 67, 67, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759737499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[1]R generation[0..12]: [34, 35, 32, 34, 35, 32, 33, 33, 34, 34, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759752733Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759768936Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[2]R flush_numbers[0..12]: [67, 69, 64, 67, 69, 64, 66, 66, 67, 67, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759784309Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[2]R generation[0..12]: [34, 35, 32, 34, 35, 32, 33, 33, 34, 34, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759802451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759819244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Max found gen is 36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759832788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Generation requested: 36 >= found:36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759847082Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs",Wait for a query_work_queue command to finish before sending IO "session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Next flush: 70","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759867373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759880757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"":"downstairs","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759894621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759907586Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"0","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.75992131Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"1","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759935124Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"client":"2","":"downstairs","session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759949368Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:50:14.759962712Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"324d3363-e93b-425c-aaaa-c8cbc1711ec7 is now active with session: b2cce334-6bbd-4858-ad7e-43825338be04","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759978535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.759991389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344,"session_id":"b2cce334-6bbd-4858-ad7e-43825338be04"} {"msg":"The guest has finished waiting for activation with:36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.760004964Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} {"msg":"Activated sub_volume 324d3363-e93b-425c-aaaa-c8cbc1711ec7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:14.760018528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 436 len 36864 data: 5 6 6 6 6 6 5 4 4 02/30 Read block 103 len 40960 03/30 Write block 428 len 32768 data: 2 4 4 4 4 4 4 5 04/30 Read block 141 len 20480 05/30 Read block 264 len 24576 06/30 Read block 200 len 4096 07/30 Read block 427 len 16384 08/30 Read block 127 len 12288 09/30 Write block 443 len 16384 data: 5 5 4 4 10/30 Write block 267 len 12288 data: 5 4 5 11/30 Read block 420 len 36864 12/30 Flush 13/30 Write block 356 len 24576 data: 5 6 6 5 4 5 14/30 Read block 412 len 28672 15/30 Read block 332 len 20480 16/30 Write block 146 len 24576 data: 3 3 5 5 6 6 17/30 Read block 230 len 16384 18/30 Flush 19/30 Write block 438 len 4096 data: 7 20/30 Write block 304 len 32768 data: 4 4 3 3 4 5 4 4 21/30 Write block 136 len 40960 data: 5 5 5 5 5 4 3 3 3 3 22/30 Read block 521 len 36864 23/30 Read block 82 len 4096 24/30 Read block 460 len 8192 25/30 Read block 90 len 8192 26/30 Write block 249 len 20480 data: 2 2 2 2 3 27/30 Read block 579 len 4096 28/30 Write block 463 len 36864 data: 4 5 5 5 7 7 7 6 5 29/30 Write block 293 len 12288 data: 4 3 3 30/30 Read block 209 len 24576 ---------------------------------------------------------------- Crucible gen:36 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Done false 1002 Acked Write 8 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Read 6 Done Done Done false 1005 Acked Read 1 Done Done Done false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 3 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 9 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Read 1 Done Done Sent false 1023 Acked Read 2 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Write 3 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 24 25 Done 30 29 6 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:14.772457827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1344} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 67 67 67 F F F 1 020-039 35 35 35 69 69 69 F F F 2 040-059 32 32 32 64 64 64 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 69 69 69 F F F 5 100-119 32 32 32 64 64 64 F F F 6 120-139 33 33 33 66 66 66 T T F <--- 7 140-159 36 36 33 71 71 66 T T F <--- 8 160-179 34 34 34 67 67 67 F F F 9 180-199 34 34 34 67 67 67 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 32 32 32 64 64 64 F F F 12 240-259 32 32 32 64 64 64 T T F <--- 13 260-279 36 36 32 70 70 64 F F F <--- 14 280-299 34 34 34 68 68 68 T T F <--- 15 300-319 30 30 30 62 62 62 T T F <--- 16 320-339 34 34 34 67 67 67 F F F 17 340-359 36 36 34 71 71 67 F F F <--- 18 360-379 36 36 35 71 71 69 F F F <--- 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 36 36 35 70 70 69 T T T <--- 22 440-459 36 36 34 70 70 67 F F T <--- 23 460-479 34 34 34 67 67 67 T T F <--- 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 36, Max flush: 71 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 18 Verifying data now No change after verify Loop: 18 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 34 34 34 67 67 67 F F F 1 020-039 35 35 35 69 69 69 F F F 2 040-059 32 32 32 64 64 64 F F F 3 060-079 34 34 34 67 67 67 F F F 4 080-099 35 35 35 69 69 69 F F F 5 100-119 32 32 32 64 64 64 F F F 6 120-139 37 37 37 72 72 72 F F F 7 140-159 37 37 37 72 72 72 F F F 8 160-179 34 34 34 67 67 67 F F F 9 180-199 34 34 34 67 67 67 F F F 10 200-219 27 27 27 53 53 53 F F F 11 220-239 32 32 32 64 64 64 F F F 12 240-259 37 37 37 72 72 72 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 37 37 37 72 72 72 F F F 15 300-319 37 37 37 72 72 72 F F F 16 320-339 34 34 34 67 67 67 F F F 17 340-359 36 36 36 71 71 71 F F F 18 360-379 36 36 36 71 71 71 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 37 37 37 72 72 72 F F F 22 440-459 36 36 36 70 70 70 F F F 23 460-479 37 37 37 72 72 72 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 37, Max flush: 72 Begin loop 19 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.088554771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.100237738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.100548683Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.100588766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.10060136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.100619642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} {"msg":"Crucible 399408a4-c49c-4806-8c1c-1811f1ee990b has session id: fe684866-2dc9-4939-bebc-c991040cb290","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.10099398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Upstairs opts: Upstairs UUID: 399408a4-c49c-4806-8c1c-1811f1ee990b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101030494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Crucible stats registered with UUID: 399408a4-c49c-4806-8c1c-1811f1ee990b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101046387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:19.101059762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101272539Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101308494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101323847Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101441776Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"task":"crutest"} {"msg":"The guest has requested activation with gen:38","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101458049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} {"msg":"399408a4-c49c-4806-8c1c-1811f1ee990b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101473952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101486786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"0","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.10150091Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"1","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101514764Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"2","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101529118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101543562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101564273Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"ds_connection connected from Ok(127.0.0.1:37534)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101604056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"2","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"ds_connection connected from Ok(127.0.0.1:59502)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101620758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"0","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"ds_connection connected from Ok(127.0.0.1:57712)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.101636292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"io task","client":"1","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102310809Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"0","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102336768Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"1","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102353691Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"2","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102668534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102721311Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[0]R flush_numbers[0..12]: [67, 69, 64, 67, 69, 64, 72, 72, 67, 67, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102784804Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[0]R generation[0..12]: [34, 35, 32, 34, 35, 32, 37, 37, 34, 34, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102802826Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102819859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[1]R flush_numbers[0..12]: [67, 69, 64, 67, 69, 64, 72, 72, 67, 67, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102835612Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[1]R generation[0..12]: [34, 35, 32, 34, 35, 32, 37, 37, 34, 34, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102850835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102867338Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[2]R flush_numbers[0..12]: [67, 69, 64, 67, 69, 64, 72, 72, 67, 67, 53, 64]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102882582Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[2]R generation[0..12]: [34, 35, 32, 34, 35, 32, 37, 37, 34, 34, 27, 32]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102897315Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102913678Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Max found gen is 38","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102927052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Generation requested: 38 >= found:38","v":Wait for a query_work_queue command to finish before sending IO 0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102947234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Next flush: 73","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102961147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102974322Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"":"downstairs","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.102988196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103003099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"0","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103017153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"1","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103031157Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"client":"2","":"downstairs","session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:50:19.103045141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103061074Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"399408a4-c49c-4806-8c1c-1811f1ee990b is now active with session: fe684866-2dc9-4939-bebc-c991040cb290","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103075948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103088922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352,"session_id":"fe684866-2dc9-4939-bebc-c991040cb290"} {"msg":"The guest has finished waiting for activation with:38","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103102546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} {"msg":"Activated sub_volume 399408a4-c49c-4806-8c1c-1811f1ee990b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:19.103114221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 315 len 28672 data: 4 4 4 3 4 3 3 02/30 Write block 168 len 32768 data: 5 6 7 8 7 5 5 5 03/30 Read block 108 len 36864 04/30 Write block 233 len 12288 data: 6 4 4 05/30 Read block 531 len 8192 06/30 Write block 134 len 40960 data: 5 6 6 6 6 6 6 5 4 4 07/30 Read block 480 len 12288 08/30 Flush 09/30 Flush 10/30 Write block 294 len 32768 data: 4 4 4 6 4 5 5 6 11/30 Read block 349 len 28672 12/30 Read block 450 len 28672 13/30 Read block 273 len 24576 14/30 Write block 449 len 36864 data: 3 3 3 3 3 4 3 4 4 15/30 Write block 49 len 12288 data: 6 6 7 16/30 Write block 58 len 16384 data: 6 5 5 4 17/30 Write block 201 len 36864 data: 4 4 4 5 5 4 3 3 4 18/30 Read block 311 len 36864 19/30 Read block 308 len 36864 20/30 Read block 590 len 12288 21/30 Flush 22/30 Write block 63 len 32768 data: 5 5 4 4 4 4 3 3 23/30 Write block 14 len 16384 data: 4 5 6 5 24/30 Read block 290 len 12288 25/30 Read block 243 len 28672 26/30 Flush 27/30 Write block 162 len 4096 data: 3 28/30 Read block 186 len 8192 29/30 Read block 333 len 40960 30/30 Read block 393 len 20480 ---------------------------------------------------------------- Crucible gen:38 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 9 Done Done Sent false 1003 Acked Write 3 Done Done Sent false 1004 Acked Read 2 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Read 5 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 28 29 Done 30 29 2 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:19.115994039Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1352} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 34 76 76 67 F F F <--- 1 020-039 35 35 35 69 69 69 F F F 2 040-059 38 38 32 75 75 64 F F F <--- 3 060-079 38 38 34 76 76 67 F F F <--- 4 080-099 35 35 35 69 69 69 F F F 5 100-119 32 32 32 64 64 64 F F F 6 120-139 38 38 37 73 73 72 F F F <--- 7 140-159 38 38 37 73 73 72 F F F <--- 8 160-179 38 38 34 73 73 67 T T T <--- 9 180-199 34 34 34 67 67 67 F F F 10 200-219 38 38 27 75 75 53 F F F <--- 11 220-239 38 38 32 73 73 64 F F F <--- 12 240-259 37 37 37 72 72 72 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 38 38 37 75 75 72 F F F <--- 15 300-319 38 38 37 75 75 72 F F T <--- 16 320-339 38 38 34 73 73 67 F F T <--- 17 340-359 36 36 36 71 71 71 F F F 18 360-379 36 36 36 71 71 71 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 36 75 75 70 F F F <--- 23 460-479 37 37 37 72 72 72 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 38, Max flush: 76 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 19 Verifying data now No change after verify Loop: 19 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 35 35 35 69 69 69 F F F 2 040-059 38 38 38 75 75 75 F F F 3 060-079 38 38 38 76 76 76 F F F 4 080-099 35 35 35 69 69 69 F F F 5 100-119 32 32 32 64 64 64 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 38 38 38 73 73 73 F F F 8 160-179 39 39 39 77 77 77 F F F 9 180-199 34 34 34 67 67 67 F F F 10 200-219 38 38 38 75 75 75 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 37 37 37 72 72 72 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 38 38 38 75 75 75 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 36 36 36 71 71 71 F F F 18 360-379 36 36 36 71 71 71 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 32 32 32 64 64 64 F F F 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 37 37 37 72 72 72 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 28 28 28 54 54 54 F F F 29 580-599 32 32 32 65 65 65 F F F Max gen: 39, Max flush: 77 Begin loop 20 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.433724987Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.445469037Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.445802302Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.445839816Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.445852051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.445870023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} {"msg":"Crucible 4c97fd78-e182-4b8a-9c43-ae30af5c560d has session id: 33f9120d-0fcd-4e30-a546-46f011e5bf86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446183267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Upstairs opts: Upstairs UUID: 4c97fd78-e182-4b8a-9c43-ae30af5c560d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446214674Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Crucible stats registered with UUID: 4c97fd78-e182-4b8a-9c43-ae30af5c560d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446232266Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:23.44624547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446479359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446507607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.44652232Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446660241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"task":"crutest"} {"msg":"The guest has requested activation with gen:40","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446678123Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} {"msg":"4c97fd78-e182-4b8a-9c43-ae30af5c560d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446699334Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446712068Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"0","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446726122Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"1","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446739966Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"2","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.44675366Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446767764Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446781998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"ds_connection connected from Ok(127.0.0.1:33786)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446833406Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"0","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"ds_connection connected from Ok(127.0.0.1:38768)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446852077Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"2","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"ds_connection connected from Ok(127.0.0.1:63861)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.446867231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"io task","client":"1","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.447579962Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"0","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.447608999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"1","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.447626072Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"2","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.447966654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.447982258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[0]R flush_numbers[0..12]: [76, 69, 75, 76, 69, 64, 73, 73, 77, 67, 75, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448019462Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[0]R generation[0..12]: [38, 35, 38, 38, 35, 32, 38, 38, 39, 34, 38, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448036544Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448052717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[1]R flush_numbers[0..12]: [76, 69, 75, 76, 69, 64, 73, 73, 77, 67, 75, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.44806807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[1]R generation[0..12]: [38, 35, 38, 38, 35, 32, 38, 38, 39, 34, 38, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448083674Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448099877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[2]R flush_numbers[0..12]: [76, 69, 75, 76, 69, 64, 73, 73, 77, 67, 75, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.44811532Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[2]R generation[0..12]: [38, 35, 38, 38, 35, 32, 38, 38, 39, 34, 38, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448133192Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448149465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Max found gen is 40","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:50:23.448162749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Generation requested: 40 >= found:40","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448180872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Next flush: 78","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448194376Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.44820755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"":"downstairs","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448221544Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448234029Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"0","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448248212Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"1","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:50:23.448262426Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"client":"2","":"downstairs","session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448278929Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448292243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"4c97fd78-e182-4b8a-9c43-ae30af5c560d is now active with session: 33f9120d-0fcd-4e30-a546-46f011e5bf86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448307337Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448320371Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360,"session_id":"33f9120d-0fcd-4e30-a546-46f011e5bf86"} {"msg":"The guest has finished waiting for activation with:40","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.448333695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} {"msg":"Activated sub_volume 4c97fd78-e182-4b8a-9c43-ae30af5c560d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:23.44834505Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 179 len 12288 data: 4 3 3 02/30 Write block 141 len 8192 data: 6 5 03/30 Read block 168 len 20480 04/30 Read block 17 len 4096 05/30 Read block 433 len 32768 06/30 Read block 293 len 36864 07/30 Flush 08/30 Write block 344 len 40960 data: 6 5 4 4 3 6 6 5 6 5 09/30 Write block 170 len 12288 data: 8 9 8 10/30 Write block 97 len 40960 data: 6 5 5 5 4 3 3 4 3 4 11/30 Read block 578 len 8192 12/30 Read block 546 len 8192 13/30 Write block 403 len 24576 data: 5 5 5 4 3 3 14/30 Write block 578 len 16384 data: 5 6 6 6 15/30 Write block 471 len 4096 data: 6 16/30 Read block 89 len 24576 17/30 Read block 211 len 8192 18/30 Read block 276 len 32768 19/30 Flush 20/30 Write block 59 len 8192 data: 6 6 21/30 Read block 184 len 24576 22/30 Read block 136 len 36864 23/30 Write block 291 len 4096 data: 7 24/30 Read block 131 len 28672 25/30 Flush 26/30 Write block 198 len 32768 data: 6 6 5 5 5 5 6 6 27/30 Read block 249 len 40960 28/30 Read block 219 len 32768 29/30 Write block 411 len 28672 data: 4 5 5 5 4 4 5 30/30 Read block 466 len 20480 ---------------------------------------------------------------- Crucible gen:40 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 1 Done Done Done false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 3 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Read 2 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Write 1 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Read 2 Done Done Sent false 1017 Acked Read 8 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 8 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 5 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 26 26 Done 30 30 4 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:23.460121327Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1360} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 35 35 35 69 69 69 F F F 2 040-059 40 40 38 80 80 75 F F F <--- 3 060-079 40 40 38 80 80 76 F F F <--- 4 080-099 40 40 35 79 79 69 F F F <--- 5 100-119 40 40 32 79 79 64 F F F <--- 6 120-139 38 38 38 73 73 73 F F F 7 140-159 40 40 38 78 78 73 F F T <--- 8 160-179 40 40 39 79 79 77 F F T <--- 9 180-199 40 40 34 78 78 67 T T T <--- 10 200-219 38 38 38 75 75 75 T T F <--- 11 220-239 38 38 38 73 73 73 F F F 12 240-259 37 37 37 72 72 72 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 40 40 38 80 80 75 F F F <--- 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 40 40 36 79 79 71 F F F <--- 18 360-379 36 36 36 71 71 71 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 40 40 32 79 79 64 T T F <--- 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 40 40 37 79 79 72 F F F <--- 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 40 40 28 79 79 54 F F F <--- 29 580-599 40 40 32 79 79 65 F F F <--- Max gen: 40, Max flush: 80 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 20 Verifying data now No change after verify Loop: 20 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 35 35 35 69 69 69 F F F 2 040-059 40 40 40 80 80 80 F F F 3 060-079 40 40 40 80 80 80 F F F 4 080-099 40 40 40 79 79 79 F F F 5 100-119 40 40 40 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 41 41 41 81 81 81 F F F 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 37 37 37 72 72 72 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 40 40 40 80 80 80 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 40 40 40 79 79 79 F F F 18 360-379 36 36 36 71 71 71 F F F 19 380-399 21 21 21 35 35 35 F F F 20 400-419 41 41 41 81 81 81 F F F 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 40 40 40 79 79 79 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 30 30 30 62 62 62 F F F 28 560-579 40 40 40 79 79 79 F F F 29 580-599 40 40 40 79 79 79 F F F Max gen: 41, Max flush: 81 Begin loop 21 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.783041276Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.794766314Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795080897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.79512161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795134114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795151627Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} {"msg":"Crucible 40829b33-5424-4fb7-8a68-2a2c0754646e has session id: 62d42591-39c8-4910-8b12-0ba3de710fc0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795505593Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Upstairs opts: Upstairs UUID: 40829b33-5424-4fb7-8a68-2a2c0754646e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795540098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Crucible stats registered with UUID: 40829b33-5424-4fb7-8a68-2a2c0754646e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795555772Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:27.795569006Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795746749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795783653Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795799846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795926041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"task":"crutest"} {"msg":"The guest has requested activation with gen:42","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795942334Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} {"msg":"40829b33-5424-4fb7-8a68-2a2c0754646e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795960256Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795976319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"0","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.795990443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"1","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796004507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"2","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796018201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796032435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796046829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:58927)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796130992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"0","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:53797)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796195974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"1","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"ds_connection connected from Ok(127.0.0.1:45776)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796212247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"io task","client":"2","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796811147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"0","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.796896011Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"2","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797101312Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"1","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797185385Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797237053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[0]R flush_numbers[0..12]: [76, 69, 80, 80, 79, 79, 73, 78, 79, 81, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797472281Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[0]R generation[0..12]: [38, 35, 40, 40, 40, 40, 38, 40, 40, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797491982Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797508815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[1]R flush_numbers[0..12]: [76, 69, 80, 80, 79, 79, 73, 78, 79, 81, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797524508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[1]R generation[0..12]: [38, 35, 40, 40, 40, 40, 38, 40, 40, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797539832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797556065Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[2]R flush_numbers[0..12]: [76, 69, 80, 80, 79, 79, 73, 78, 79, 81, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797571398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[2]R generation[0..12]: [38, 35, 40, 40, 40, 40, 38, 40, 40, 41, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797587071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797604164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Max found gen is 42","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:50:27.797618158Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Generation requested: 42 >= found:42","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797637209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Next flush: 82","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797650564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797664048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"":"downstairs","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797678472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797691326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"0","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797707109Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"1","":"downstairs"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797725981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"client":"2","":"downstairs","session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797740005Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797753539Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"40829b33-5424-4fb7-8a68-2a2c0754646e is now active with session: 62d42591-39c8-4910-8b12-0ba3de710fc0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797768233Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797780537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368,"session_id":"62d42591-39c8-4910-8b12-0ba3de710fc0"} {"msg":"The guest has finished waiting for activation with:42","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797793642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} {"msg":"Activated sub_volume 40829b33-5424-4fb7-8a68-2a2c0754646e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:27.797805097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 349 len 16384 data: 7 7 6 7 02/30 Write block 392 len 12288 data: 4 4 2 03/30 Read block 591 len 4096 04/30 Write block 250 len 16384 data: 3 3 3 4 05/30 Flush 06/30 Flush 07/30 Write block 364 len 4096 data: 4 08/30 Read block 81 len 12288 09/30 Flush 10/30 Write block 542 len 8192 data: 2 2 11/30 Read block 16 len 12288 12/30 Read block 346 len 24576 13/30 Read block 518 len 4096 14/30 Write block 289 len 4096 data: 9 15/30 Flush 16/30 Write block 186 len 12288 data: 4 4 3 17/30 Write block 287 len 24576 data: 10 9 10 7 8 6 18/30 Write block 374 len 8192 data: 2 2 19/30 Read block 468 len 20480 20/30 Read block 47 len 4096 21/30 Write block 355 len 20480 data: 5 6 7 7 6 22/30 Write block 23 len 8192 data: 4 5 23/30 Read block 283 len 16384 24/30 Read block 7 len 4096 25/30 Read block 9 len 32768 26/30 Read block 398 len 4096 27/30 Read block 446 len 16384 28/30 Write block 419 len 4096 data: 4 29/30 Read block 339 len 32768 30/30 Write block 359 len 20480 data: 7 5 6 3 4 ---------------------------------------------------------------- Crucible gen:42 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Sent Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Read 1 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Flush 0 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 2 Done Sent Done false 1010 Acked Read 3 Done Sent Done false 1011 Acked Read 6 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Write 2 Done Sent Done false 1022 Acked Read 4 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 1 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1014)) None Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:27.807507002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1368} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 35 35 35 69 69 69 T F T <--- 2 040-059 40 40 40 80 80 80 F F F 3 060-079 40 40 40 80 80 80 F F F 4 080-099 40 40 40 79 79 79 F F F 5 100-119 40 40 40 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 41 41 41 81 81 81 T F T <--- 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 42 37 42 82 72 82 F F F <--- 13 260-279 36 36 36 70 70 70 F F F 14 280-299 42 40 42 85 80 85 T F T <--- 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 42 40 42 82 79 82 T F T <--- 18 360-379 42 36 42 84 71 84 T F T <--- 19 380-399 42 21 42 82 35 82 F F F <--- 20 400-419 41 41 41 81 81 81 T F T <--- 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 40 40 40 79 79 79 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 42 30 42 85 62 85 F F F <--- 28 560-579 40 40 40 79 79 79 F F F 29 580-599 40 40 40 79 79 79 F F F Max gen: 42, Max flush: 85 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 21 Verifying data now No change after verify Loop: 21 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 43 43 43 86 86 86 F F F 2 040-059 40 40 40 80 80 80 F F F 3 060-079 40 40 40 80 80 80 F F F 4 080-099 40 40 40 79 79 79 F F F 5 100-119 40 40 40 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 43 43 43 86 86 86 F F F 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 42 42 42 82 82 82 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 43 43 43 86 86 86 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 43 43 43 86 86 86 F F F 18 360-379 43 43 43 86 86 86 F F F 19 380-399 42 42 42 82 82 82 F F F 20 400-419 43 43 43 86 86 86 F F F 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 40 40 40 79 79 79 F F F 24 480-499 32 32 32 64 64 64 F F F 25 500-519 32 32 32 64 64 64 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 42 42 42 85 85 85 F F F 28 560-579 40 40 40 79 79 79 F F F 29 580-599 40 40 40 79 79 79 F F F Max gen: 43, Max flush: 86 Begin loop 22 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.119689885Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.131342335Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.131666365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.131707737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.131720801Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.131738763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} {"msg":"Crucible d5001f4f-2fb3-43b4-89f6-fc143aae084f has session id: deb3e14d-dfb2-4111-b014-d943f70b716c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132066481Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Upstairs opts: Upstairs UUID: d5001f4f-2fb3-43b4-89f6-fc143aae084f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.13213828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Crucible stats registered with UUID: d5001f4f-2fb3-43b4-89f6-fc143aae084f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132166288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:32.132180622Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132349299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132385143Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132400726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132503222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"task":"crutest"} {"msg":"The guest has requested activation with gen:44","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132520445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} {"msg":"d5001f4f-2fb3-43b4-89f6-fc143aae084f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132537637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132550392Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"0","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132564605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"1","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132578569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"2","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132592293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132607017Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132621771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46080)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132657915Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"2","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40453)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132674378Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"1","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"ds_connection connected from Ok(127.0.0.1:46601)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.132788928Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"io task","client":"0","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133302686Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"1","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133349585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"0","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133484327Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"2","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133650605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133689128Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[0]R flush_numbers[0..12]: [76, 86, 80, 80, 79, 79, 73, 78, 79, 86, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133850178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[0]R generation[0..12]: [38, 43, 40, 40, 40, 40, 38, 40, 40, 43, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.13386855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133885503Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[1]R flush_numbers[0..12]: [76, 86, 80, 80, 79, 79, 73, 78, 79, 86, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133901246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[1]R generation[0..12]: [38, 43, 40, 40, 40, 40, 38, 40, 40, 43, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133917989Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133935561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[2]R flush_numbers[0..12]: [76, 86, 80, 80, 79, 79, 73, 78, 79, 86, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133951334Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[2]R generation[0..12]: [38, 43, 40, 40, 40, 40, 38, 40, 40, 43, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133966798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.133983231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 44","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134004991Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Generation requested: 44 >= found:44","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134019515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Next flush: 87","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134032729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134046063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"":"downstairs","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134060187Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134072852Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"0","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:50:32.134086886Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"1","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134103768Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"client":"2","":"downstairs","session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134118502Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134131816Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"d5001f4f-2fb3-43b4-89f6-fc143aae084f is now active with session: deb3e14d-dfb2-4111-b014-d943f70b716c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.13414656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134159244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376,"session_id":"deb3e14d-dfb2-4111-b014-d943f70b716c"} {"msg":"The guest has finished waiting for activation with:44","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134172279Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} {"msg":"Activated sub_volume d5001f4f-2fb3-43b4-89f6-fc143aae084f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:32.134183764Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 88 len 8192 data: 4 4 02/30 Flush 03/30 Write block 257 len 8192 data: 6 7 04/30 Write block 94 len 4096 data: 7 05/30 Write block 568 len 20480 data: 3 3 2 2 2 06/30 Write block 506 len 24576 data: 3 3 4 4 4 4 07/30 Read block 167 len 20480 08/30 Read block 112 len 28672 09/30 Read block 283 len 12288 10/30 Read block 438 len 8192 11/30 Flush 12/30 Write block 399 len 32768 data: 2 3 4 5 6 6 6 5 13/30 Read block 501 len 28672 14/30 Read block 361 len 20480 15/30 Write block 406 len 24576 data: 6 4 4 5 4 5 16/30 Flush 17/30 Write block 344 len 12288 data: 7 6 5 18/30 Read block 452 len 24576 19/30 Flush 20/30 Write block 489 len 32768 data: 6 4 3 3 3 3 4 5 21/30 Flush 22/30 Write block 367 len 4096 data: 3 23/30 Read block 429 len 12288 24/30 Write block 54 len 12288 data: 7 7 6 25/30 Write block 20 len 8192 data: 2 2 26/30 Read block 553 len 36864 27/30 Read block 192 len 36864 28/30 Write block 380 len 24576 data: 4 5 4 3 3 4 29/30 Read block 440 len 20480 30/30 Write block 288 len 4096 data: 10 ---------------------------------------------------------------- Crucible gen:44 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Sent false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 2 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Write 6 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Flush 0 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 5 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Read 6 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 3 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 9 Done Done Sent false 1027 Acked Write 6 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) Some(JobId(1020)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:32.145147942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1376} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 43 43 43 86 86 86 T T F <--- 2 040-059 40 40 40 80 80 80 T T F <--- 3 060-079 40 40 40 80 80 80 F F F 4 080-099 44 44 40 88 88 79 F F F <--- 5 100-119 40 40 40 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 43 43 43 86 86 86 F F F 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 44 42 88 88 82 F F F <--- 13 260-279 36 36 36 70 70 70 F F F 14 280-299 43 43 43 86 86 86 T T F <--- 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 44 44 43 90 90 86 F F F <--- 18 360-379 43 43 43 86 86 86 T T F <--- 19 380-399 44 44 42 89 89 82 T T F <--- 20 400-419 44 44 43 89 89 86 F F F <--- 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 40 40 40 79 79 79 F F F 24 480-499 44 44 32 91 91 64 F F F <--- 25 500-519 44 44 32 88 88 64 F F F <--- 26 520-539 35 35 35 69 69 69 F F F 27 540-559 42 42 42 85 85 85 F F F 28 560-579 44 44 40 88 88 79 F F F <--- 29 580-599 40 40 40 79 79 79 F F F Max gen: 44, Max flush: 91 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 22 Verifying data now No change after verify Loop: 22 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 45 45 45 92 92 92 F F F 2 040-059 45 45 45 92 92 92 F F F 3 060-079 40 40 40 80 80 80 F F F 4 080-099 44 44 44 88 88 88 F F F 5 100-119 40 40 40 79 79 79 F F F 6 120-139 38 38 38 73 73 73 F F F 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 43 43 43 86 86 86 F F F 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 44 44 88 88 88 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 45 45 45 92 92 92 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 44 44 44 90 90 90 F F F 18 360-379 45 45 45 92 92 92 F F F 19 380-399 45 45 45 92 92 92 F F F 20 400-419 44 44 44 89 89 89 F F F 21 420-439 37 37 37 72 72 72 F F F 22 440-459 38 38 38 75 75 75 F F F 23 460-479 40 40 40 79 79 79 F F F 24 480-499 44 44 44 91 91 91 F F F 25 500-519 44 44 44 88 88 88 F F F 26 520-539 35 35 35 69 69 69 F F F 27 540-559 42 42 42 85 85 85 F F F 28 560-579 44 44 44 88 88 88 F F F 29 580-599 40 40 40 79 79 79 F F F Max gen: 45, Max flush: 92 Begin loop 23 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.463306994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475015689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475330723Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475371795Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475386788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475404771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} {"msg":"Crucible 886d61b9-9662-4017-ae69-1dc18f26da89 has session id: 8d4ff6b3-765e-4ac2-95c0-1404ef94848e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475757208Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Upstairs opts: Upstairs UUID: 886d61b9-9662-4017-ae69-1dc18f26da89, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475792632Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Crucible stats registered with UUID: 886d61b9-9662-4017-ae69-1dc18f26da89","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475808056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:36.47582121Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.475995554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476024752Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476039485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47616831Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"task":"crutest"} {"msg":"The guest has requested activation with gen:46","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476186452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} {"msg":"886d61b9-9662-4017-ae69-1dc18f26da89 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476203504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476216409Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"0","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476230543Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"1","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476246666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"2","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47626037Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476274953Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476293345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"ds_connection connected from Ok(127.0.0.1:64724)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476308799Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"1","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"ds_connection connected from Ok(127.0.0.1:52838)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476323472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"2","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48865)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.476478255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"io task","client":"0","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477098986Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"0","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47715843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"1","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477281047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"2","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477490457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47752913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[0]R flush_numbers[0..12]: [76, 92, 92, 80, 88, 79, 73, 78, 79, 86, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47764484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[0]R generation[0..12]: [38, 45, 45, 40, 44, 40, 38, 40, 40, 43, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477661722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477678235Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[1]R flush_numbers[0..12]: [76, 92, 92, 80, 88, 79, 73, 78, 79, 86, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477697177Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[1]R generation[0..12]: [38, 45, 45, 40, 44, 40, 38, 40, 40, 43, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47771253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477728743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[2]R flush_numbers[0..12]: [76, 92, 92, 80, 88, 79, 73, 78, 79, 86, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477744277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[2]R generation[0..12]: [38, 45, 45, 40, 44, 40, 38, 40, 40, 43, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47775988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477776413Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Max found gen is 46","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:50:36.477790197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Generation requested: 46 >= found:46","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477809988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Next flush: 93","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477823212Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477836846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"":"downstairs","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47785154Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477864574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"0","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477879008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"1","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Transition from WaitQuorum to ActiveDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477896661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"client":"2","":"downstairs","session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477913843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477927277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"886d61b9-9662-4017-ae69-1dc18f26da89 is now active with session: 8d4ff6b3-765e-4ac2-95c0-1404ef94848e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477942221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477955025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384,"session_id":"8d4ff6b3-765e-4ac2-95c0-1404ef94848e"} {"msg":"The guest has finished waiting for activation with:46","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.47796821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} {"msg":"Activated sub_volume 886d61b9-9662-4017-ae69-1dc18f26da89","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:36.477979974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 78 len 4096 data: 4 02/30 Read block 529 len 24576 03/30 Read block 99 len 12288 04/30 Flush 05/30 Write block 532 len 36864 data: 4 4 3 2 2 2 2 3 3 06/30 Read block 482 len 36864 07/30 Read block 331 len 16384 08/30 Write block 108 len 28672 data: 4 4 4 4 5 5 5 09/30 Flush 10/30 Write block 586 len 12288 data: 5 4 4 11/30 Write block 388 len 4096 data: 4 12/30 Write block 435 len 28672 data: 6 6 7 8 7 7 7 13/30 Read block 366 len 24576 14/30 Read block 193 len 28672 15/30 Write block 451 len 12288 data: 4 4 4 16/30 Write block 579 len 36864 data: 7 7 7 5 4 4 5 6 5 17/30 Write block 125 len 24576 data: 8 5 4 4 4 3 18/30 Read block 407 len 4096 19/30 Read block 77 len 12288 20/30 Write block 581 len 28672 data: 8 6 5 5 6 7 6 21/30 Read block 23 len 36864 22/30 Write block 293 len 24576 data: 5 5 5 5 7 5 23/30 Write block 33 len 36864 data: 5 5 5 6 3 3 3 4 4 24/30 Write block 488 len 8192 data: 6 7 25/30 Write block 194 len 4096 data: 6 26/30 Flush 27/30 Write block 119 len 36864 data: 3 5 5 6 7 6 9 6 5 28/30 Write block 182 len 20480 data: 3 3 2 3 5 29/30 Write block 418 len 36864 data: 5 5 4 4 5 5 5 4 3 30/30 Read block 420 len 28672 ---------------------------------------------------------------- Crucible gen:46 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 6 Done Done Sent false 1002 Acked Read 3 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Write 7 Done Done Sent false 1012 Acked Read 6 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Write 3 Done Done Sent false 1015 Acked Write 9 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 1 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 7 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Write 9 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 29 31 Done 30 28 1 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:36.490446971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1384} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 46 46 45 95 95 92 F F F <--- 2 040-059 46 46 45 95 95 92 F F F <--- 3 060-079 46 46 40 93 93 80 F F T <--- 4 080-099 44 44 44 88 88 88 F F F 5 100-119 46 46 40 94 94 79 T T F <--- 6 120-139 46 46 38 95 95 73 T T F <--- 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 46 46 43 95 95 86 T T F <--- 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 44 44 88 88 88 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 46 46 45 95 95 92 F F F <--- 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 44 44 44 90 90 90 F F F 18 360-379 45 45 45 92 92 92 F F F 19 380-399 46 46 45 95 95 92 F F F <--- 20 400-419 44 44 44 89 89 89 T T F <--- 21 420-439 46 46 37 95 95 72 T T F <--- 22 440-459 46 46 38 95 95 75 F F F <--- 23 460-479 40 40 40 79 79 79 F F F 24 480-499 46 46 44 95 95 91 F F F <--- 25 500-519 44 44 44 88 88 88 F F F 26 520-539 46 46 35 94 94 69 F F F <--- 27 540-559 46 46 42 94 94 85 F F F <--- 28 560-579 46 46 44 95 95 88 F F F <--- 29 580-599 46 46 40 95 95 79 F F F <--- Max gen: 46, Max flush: 95 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 23 Verifying data now No change after verify Loop: 23 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 46 46 46 95 95 95 F F F 2 040-059 46 46 46 95 95 95 F F F 3 060-079 46 46 46 93 93 93 F F F 4 080-099 44 44 44 88 88 88 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 40 40 40 78 78 78 F F F 8 160-179 40 40 40 79 79 79 F F F 9 180-199 47 47 47 96 96 96 F F F 10 200-219 41 41 41 81 81 81 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 44 44 88 88 88 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 46 46 46 95 95 95 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 F F F 17 340-359 44 44 44 90 90 90 F F F 18 360-379 45 45 45 92 92 92 F F F 19 380-399 46 46 46 95 95 95 F F F 20 400-419 47 47 47 96 96 96 F F F 21 420-439 47 47 47 96 96 96 F F F 22 440-459 46 46 46 95 95 95 F F F 23 460-479 40 40 40 79 79 79 F F F 24 480-499 46 46 46 95 95 95 F F F 25 500-519 44 44 44 88 88 88 F F F 26 520-539 46 46 46 94 94 94 F F F 27 540-559 46 46 46 94 94 94 F F F 28 560-579 46 46 46 95 95 95 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 47, Max flush: 96 Begin loop 24 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.817444233Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829197579Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829516421Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829554164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829566469Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829584611Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} {"msg":"Crucible b1af3858-5b5a-4757-84e3-97970ad3d0b2 has session id: c2c48fdd-8237-40e7-8f94-712f826dc148","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829946015Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Upstairs opts: Upstairs UUID: b1af3858-5b5a-4757-84e3-97970ad3d0b2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.82998071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Crucible stats registered with UUID: b1af3858-5b5a-4757-84e3-97970ad3d0b2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.829996633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:40.830012616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830189499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830215278Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830229942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830358256Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"task":"crutest"} {"msg":"The guest has requested activation with gen:48","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830375548Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} {"msg":"b1af3858-5b5a-4757-84e3-97970ad3d0b2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830392241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830404666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"0","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.83041845Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"1","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830432604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"2","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830463011Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830478024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830492088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"ds_connection connected from Ok(127.0.0.1:40906)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830518506Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"2","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"ds_connection connected from Ok(127.0.0.1:58447)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.83053332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"0","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"ds_connection connected from Ok(127.0.0.1:47986)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.830654887Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"io task","client":"1","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.831323857Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"0","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.831357163Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"1","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.831622308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"2","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.83171068Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.831726063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[0]R flush_numbers[0..12]: [76, 95, 95, 93, 88, 96, 96, 78, 79, 96, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832020186Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[0]R generation[0..12]: [38, 46, 46, 46, 44, 47, 47, 40, 40, 47, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832037038Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832053641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[1]R flush_numbers[0..12]: [76, 95, 95, 93, 88, 96, 96, 78, 79, 96, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832068885Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[1]R generation[0..12]: [38, 46, 46, 46, 44, 47, 47, 40, 40, 47, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832084148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832100401Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[2]R flush_numbers[0..12]: [76, 95, 95, 93, 88, 96, 96, 78, 79, 96, 81, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832115644Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[2]R generation[0..12]: [38, 46, 46, 46, 44, 47, 47, 40, 40, 47, 41, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832130928Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832147331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Max found gen is 48","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832163494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Generation requested: 48 >= found:48","v":0,"Wait for a query_work_queue command to finish before sending IO name":"crucible","level":30,"time":"2026-03-12T22:50:40.832183565Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Next flush: 97","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832197229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832210393Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"":"downstairs","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832224167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832236652Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"0","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832250976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"1","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832265209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"client":"2","":"downstairs","session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832279633Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832296236Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"b1af3858-5b5a-4757-84e3-97970ad3d0b2 is now active with session: c2c48fdd-8237-40e7-8f94-712f826dc148","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832311559Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832324204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392,"session_id":"c2c48fdd-8237-40e7-8f94-712f826dc148"} {"msg":"The guest has finished waiting for activation with:48","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832337518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} {"msg":"Activated sub_volume b1af3858-5b5a-4757-84e3-97970ad3d0b2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:40.832349013Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 471 len 20480 data: 7 4 4 4 3 03/30 Write block 515 len 40960 data: 4 4 5 4 4 4 4 3 3 3 04/30 Read block 76 len 24576 05/30 Write block 512 len 36864 data: 4 4 4 5 5 6 5 5 5 06/30 Read block 324 len 12288 07/30 Write block 541 len 8192 data: 2 3 08/30 Flush 09/30 Write block 203 len 36864 data: 6 7 7 5 4 4 5 3 4 10/30 Write block 323 len 20480 data: 3 3 2 3 4 11/30 Write block 29 len 4096 data: 6 12/30 Read block 247 len 4096 13/30 Write block 31 len 40960 data: 5 5 6 6 6 7 4 4 4 5 14/30 Write block 380 len 28672 data: 5 6 5 4 4 5 4 15/30 Write block 176 len 40960 data: 4 3 3 5 4 4 4 4 3 4 16/30 Read block 90 len 4096 17/30 Write block 510 len 16384 data: 5 5 5 5 18/30 Write block 452 len 24576 data: 5 5 5 4 5 5 19/30 Read block 192 len 36864 20/30 Write block 339 len 40960 data: 4 4 6 7 7 8 7 6 5 4 21/30 Read block 516 len 32768 22/30 Read block 265 len 40960 23/30 Write block 335 len 40960 data: 4 5 5 6 5 5 7 8 8 9 24/30 Write block 92 len 8192 data: 7 7 25/30 Read block 36 len 12288 26/30 Read block 584 len 36864 27/30 Read block 145 len 12288 28/30 Read block 167 len 24576 29/30 Write block 159 len 28672 data: 3 3 3 4 3 2 2 30/30 Read block 358 len 36864 ---------------------------------------------------------------- Crucible gen:48 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 6 Done Done Done false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Write 5 Done Done Sent false 1010 Acked Write 1 Done Done Sent false 1011 Acked Read 1 Done Done Sent false 1012 Acked Write 10 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 10 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 9 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 8 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 2 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 3 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 26 27 Done 29 28 3 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:40.845361843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1392} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 46 46 46 95 95 95 T T F <--- 2 040-059 46 46 46 95 95 95 T T F <--- 3 060-079 46 46 46 93 93 93 F F F 4 080-099 44 44 44 88 88 88 T T F <--- 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 40 40 40 78 78 78 T T F <--- 8 160-179 40 40 40 79 79 79 T T F <--- 9 180-199 47 47 47 96 96 96 T T F <--- 10 200-219 41 41 41 81 81 81 T T F <--- 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 44 44 88 88 88 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 46 46 46 95 95 95 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 38 38 38 73 73 73 T T F <--- 17 340-359 44 44 44 90 90 90 T T F <--- 18 360-379 45 45 45 92 92 92 F F F 19 380-399 46 46 46 95 95 95 T T F <--- 20 400-419 47 47 47 96 96 96 F F F 21 420-439 47 47 47 96 96 96 F F F 22 440-459 46 46 46 95 95 95 T T F <--- 23 460-479 48 48 40 98 98 79 F F T <--- 24 480-499 46 46 46 95 95 95 F F F 25 500-519 48 48 44 98 98 88 T T T <--- 26 520-539 48 48 46 98 98 94 F F T <--- 27 540-559 48 48 46 98 98 94 F F F <--- 28 560-579 46 46 46 95 95 95 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 48, Max flush: 98 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 24 Verifying data now No change after verify Loop: 24 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 49 49 49 99 99 99 F F F 2 040-059 49 49 49 99 99 99 F F F 3 060-079 46 46 46 93 93 93 F F F 4 080-099 49 49 49 99 99 99 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 49 49 49 99 99 99 F F F 8 160-179 49 49 49 99 99 99 F F F 9 180-199 49 49 49 99 99 99 F F F 10 200-219 49 49 49 99 99 99 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 44 44 88 88 88 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 46 46 46 95 95 95 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 49 49 49 99 99 99 F F F 17 340-359 49 49 49 99 99 99 F F F 18 360-379 45 45 45 92 92 92 F F F 19 380-399 49 49 49 99 99 99 F F F 20 400-419 47 47 47 96 96 96 F F F 21 420-439 47 47 47 96 96 96 F F F 22 440-459 49 49 49 99 99 99 F F F 23 460-479 48 48 48 98 98 98 F F F 24 480-499 46 46 46 95 95 95 F F F 25 500-519 49 49 49 99 99 99 F F F 26 520-539 48 48 48 98 98 98 F F F 27 540-559 48 48 48 98 98 98 F F F 28 560-579 46 46 46 95 95 95 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 49, Max flush: 99 Begin loop 25 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.178494026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:45.189235601Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.189276643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.19966753Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.19996735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200007672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200019837Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200037629Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} {"msg":"Crucible 27e598a0-b310-4feb-bc1a-2e7090978360 has session id: 7fa66819-ab5e-48e4-a2f4-d252e270eafc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200408399Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Upstairs opts: Upstairs UUID: 27e598a0-b310-4feb-bc1a-2e7090978360, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200444793Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Crucible stats registered with UUID: 27e598a0-b310-4feb-bc1a-2e7090978360","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200460446Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:45.20047423Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200662418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200698453Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200714126Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200826817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"task":"crutest"} {"msg":"The guest has requested activation with gen:50","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.20084365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} {"msg":"27e598a0-b310-4feb-bc1a-2e7090978360 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200860023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200872927Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"0","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200887041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"1","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200900955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"2","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200914619Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200928683Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200942817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"ds_connection connected from Ok(127.0.0.1:48638)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200957401Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"1","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37105)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.200975323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"0","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"ds_connection connected from Ok(127.0.0.1:54473)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.201116362Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"io task","client":"2","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.201683936Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"1","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.201730446Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"2","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.201747808Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"0","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202081124Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202109551Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[0]R flush_numbers[0..12]: [76, 99, 99, 93, 99, 96, 96, 99, 99, 99, 99, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202187348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[0]R generation[0..12]: [38, 49, 49, 46, 49, 47, 47, 49, 49, 49, 49, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202206829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202223402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[1]R flush_numbers[0..12]: [76, 99, 99, 93, 99, 96, 96, 99, 99, 99, 99, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202238685Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[1]R generation[0..12]: [38, 49, 49, 46, 49, 47, 47, 49, 49, 49, 49, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202253829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202270172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[2]R flush_numbers[0..12]: [76, 99, 99, 93, 99, 96, 96, 99, 99, 99, 99, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202285035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[2]R generation[0..12]: [38, 49, 49, 46, 49, 47, 47, 49, 49, 49, 49, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202300059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202316112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Max found gen is 50","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202329356Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Generation requested: 50 >= found:50","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202348528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Next flush: 100","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202361672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202375066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"":"downstairs","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.20238901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202401615Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"0","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202417008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"1","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202433591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"client":"2","":"downstairs","session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202447645Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202460869Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"27e598a0-b310-4feb-bc1a-2e7090978360 is now active with session: 7fa66819-ab5e-48e4-a2f4-d252e270eafc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202475773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202488247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400,"session_id":"7fa66819-ab5e-48e4-a2f4-d252e270eafc"} {"msg":"The guest has finished waiting for activation with:50","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202501082Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} {"msg":"Activated sub_volume 27e598a0-b310-4feb-bc1a-2e7090978360","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:45.202512717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 499 len 4096 data: 6 02/30 Flush 03/30 Write block 516 len 8192 data: 6 7 04/30 Read block 428 len 4096 05/30 Write block 425 len 8192 data: 5 4 06/30 Write block 511 len 40960 data: 6 6 6 5 6 7 8 6 6 6 07/30 Flush 08/30 Write block 254 len 16384 data: 4 4 4 7 09/30 Read block 100 len 40960 10/30 Write block 212 len 24576 data: 4 3 3 2 2 2 11/30 Read block 388 len 36864 12/30 Read block 189 len 28672 13/30 Read block 459 len 24576 14/30 Write block 478 len 20480 data: 3 4 5 5 7 15/30 Write block 79 len 36864 data: 4 4 4 5 5 6 7 7 5 16/30 Write block 480 len 40960 data: 6 6 8 7 6 5 5 5 7 8 17/30 Read block 240 len 24576 18/30 Write block 160 len 40960 data: 4 4 5 4 3 3 3 4 6 7 19/30 Read block 99 len 20480 20/30 Write block 404 len 40960 data: 7 7 7 5 5 6 5 6 6 6 21/30 Write block 39 len 12288 data: 5 6 5 22/30 Write block 282 len 40960 data: 4 3 6 7 10 11 11 11 8 9 23/30 Write block 569 len 8192 data: 4 3 24/30 Write block 193 len 32768 data: 6 7 5 7 8 7 7 6 25/30 Read block 18 len 40960 26/30 Flush 27/30 Write block 199 len 16384 data: 8 7 6 6 28/30 Write block 504 len 36864 data: 4 3 4 4 5 5 6 7 7 29/30 Write block 190 len 36864 data: 4 5 5 7 8 6 8 9 8 30/30 Write block 389 len 24576 data: 4 4 4 5 5 3 ---------------------------------------------------------------- Crucible gen:50 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Flush 0 Sent Done Done false 1002 Acked Write 2 Sent Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Write 10 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 4 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 9 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 6 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 5 Sent Done Done false 1019 Acked Write 10 Sent Done Done false 1020 Acked Write 3 Sent Done Done false 1021 Acked Write 10 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:45.215039647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1400} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 49 50 50 99 102 102 F F F <--- 2 040-059 49 50 50 99 102 102 F F F <--- 3 060-079 46 50 50 93 102 102 F F F <--- 4 080-099 49 50 50 99 102 102 F F F <--- 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 49 49 49 99 99 99 F F F 8 160-179 49 50 50 99 102 102 F F F <--- 9 180-199 49 50 50 99 102 102 F T T <--- 10 200-219 49 50 50 99 102 102 F T T <--- 11 220-239 38 38 38 73 73 73 F F F 12 240-259 44 50 50 88 102 102 F F F <--- 13 260-279 36 36 36 70 70 70 F F F 14 280-299 46 50 50 95 102 102 F F F <--- 15 300-319 38 38 38 75 75 75 F F F 16 320-339 49 49 49 99 99 99 F F F 17 340-359 49 49 49 99 99 99 F F F 18 360-379 45 45 45 92 92 92 F F F 19 380-399 49 49 49 99 99 99 F T T <--- 20 400-419 47 50 50 96 102 102 F F F <--- 21 420-439 47 50 50 96 101 101 F F F <--- 22 440-459 49 49 49 99 99 99 F F F 23 460-479 48 50 50 98 102 102 F F F <--- 24 480-499 46 50 50 95 102 102 F F F <--- 25 500-519 49 50 50 99 101 101 F T T <--- 26 520-539 48 50 50 98 101 101 F F F <--- 27 540-559 48 48 48 98 98 98 F F F 28 560-579 46 50 50 95 102 102 F F F <--- 29 580-599 46 46 46 95 95 95 F F F Max gen: 50, Max flush: 102 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 25 Verifying data now No change after verify Loop: 25 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 50 50 50 102 102 102 F F F 2 040-059 50 50 50 102 102 102 F F F 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 49 49 49 99 99 99 F F F 8 160-179 50 50 50 102 102 102 F F F 9 180-199 51 51 51 103 103 103 F F F 10 200-219 51 51 51 103 103 103 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 50 50 50 102 102 102 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 50 50 50 102 102 102 F F F 15 300-319 38 38 38 75 75 75 F F F 16 320-339 49 49 49 99 99 99 F F F 17 340-359 49 49 49 99 99 99 F F F 18 360-379 45 45 45 92 92 92 F F F 19 380-399 51 51 51 103 103 103 F F F 20 400-419 50 50 50 102 102 102 F F F 21 420-439 50 50 50 101 101 101 F F F 22 440-459 49 49 49 99 99 99 F F F 23 460-479 50 50 50 102 102 102 F F F 24 480-499 50 50 50 102 102 102 F F F 25 500-519 51 51 51 103 103 103 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 48 48 48 98 98 98 F F F 28 560-579 50 50 50 102 102 102 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 51, Max flush: 103 Begin loop 26 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.556560434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.568474401Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.568794372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.568834885Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.568847149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.568864862Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} {"msg":"Crucible 4dbfce9b-b092-464e-b883-5e7de2a8e731 has session id: 742f5dee-cc7f-481d-8cf9-737d803e1468","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569210812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Upstairs opts: Upstairs UUID: 4dbfce9b-b092-464e-b883-5e7de2a8e731, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569288398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Crucible stats registered with UUID: 4dbfce9b-b092-464e-b883-5e7de2a8e731","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569304421Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:49.569317615Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.56951589Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569551954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569567587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569711545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"task":"crutest"} {"msg":"The guest has requested activation with gen:52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569730277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} {"msg":"4dbfce9b-b092-464e-b883-5e7de2a8e731 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.56974519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569758115Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"0","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569772148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"1","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569785992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"2","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569799507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.56981365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569828054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"ds_connection connected from Ok(127.0.0.1:52908)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569892186Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"1","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"ds_connection connected from Ok(127.0.0.1:60067)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.569923533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"2","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"ds_connection connected from Ok(127.0.0.1:50477)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.570067351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"io task","client":"0","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.570642771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"0","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.570674697Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"1","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.57069096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"2","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571008403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571053283Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[0]R flush_numbers[0..12]: [76, 102, 102, 102, 102, 96, 96, 99, 102, 103, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571116306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[0]R generation[0..12]: [38, 50, 50, 50, 50, 47, 47, 49, 50, 51, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571133209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571149522Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[1]R flush_numbers[0..12]: [76, 102, 102, 102, 102, 96, 96, 99, 102, 103, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571165035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[1]R generation[0..12]: [38, 50, 50, 50, 50, 47, 47, 49, 50, 51, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571180278Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571196411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[2]R flush_numbers[0..12]: [76, 102, 102, 102, 102, 96, 96, 99, 102, 103, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571211835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[2]R generation[0..12]: [38, 50, 50, 50, 50, 47, 47, 49, 50, 51, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571230137Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.57124638Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Max found gen is 52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571260154Z",Wait for a query_work_queue command to finish before sending IO "hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Generation requested: 52 >= found:52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571280675Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Next flush: 104","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571293769Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571307123Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"":"downstairs","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571321167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571333622Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"0","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571347776Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"1","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2026-03-12T22:50:49.571361949Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"client":"2","":"downstairs","session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571378942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571392416Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"4dbfce9b-b092-464e-b883-5e7de2a8e731 is now active with session: 742f5dee-cc7f-481d-8cf9-737d803e1468","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.57140705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571419535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408,"session_id":"742f5dee-cc7f-481d-8cf9-737d803e1468"} {"msg":"The guest has finished waiting for activation with:52","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571432839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} {"msg":"Activated sub_volume 4dbfce9b-b092-464e-b883-5e7de2a8e731","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:49.571446133Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 504 len 12288 data: 5 4 5 02/30 Read block 523 len 4096 03/30 Flush 04/30 Write block 547 len 8192 data: 4 4 05/30 Read block 122 len 12288 06/30 Flush 07/30 Write block 452 len 40960 data: 6 6 6 5 6 6 4 4 4 5 08/30 Flush 09/30 Write block 141 len 8192 data: 7 6 10/30 Read block 123 len 20480 11/30 Read block 423 len 16384 12/30 Read block 335 len 40960 13/30 Read block 372 len 40960 14/30 Write block 160 len 12288 data: 5 5 6 15/30 Read block 276 len 4096 16/30 Read block 402 len 20480 17/30 Write block 317 len 16384 data: 5 4 5 4 18/30 Read block 287 len 40960 19/30 Read block 95 len 8192 20/30 Read block 580 len 24576 21/30 Read block 137 len 28672 22/30 Read block 296 len 20480 23/30 Write block 419 len 8192 data: 6 5 24/30 Read block 72 len 28672 25/30 Write block 574 len 20480 data: 2 2 3 3 6 26/30 Write block 148 len 20480 data: 6 6 7 7 4 27/30 Write block 405 len 40960 data: 8 8 6 6 7 6 7 7 7 6 28/30 Write block 361 len 40960 data: 7 4 5 5 3 3 4 3 4 4 29/30 Write block 406 len 16384 data: 9 7 7 8 30/30 Read block 64 len 36864 ---------------------------------------------------------------- Crucible gen:52 GIO:true work queues: Upstairs:0 downstairs:24 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1006 Acked Write 10 Done Done Done false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Read 4 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 10 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 5 Done Done Sent false 1025 Acked Write 5 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 23 24 Done 24 23 1 48 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1007)) Some(JobId(1007)) Some(JobId(1005)) Downstairs last five completed: 1005 1004 1003 1002 1001 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:49.582888914Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1408} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 50 50 50 102 102 102 F F F 2 040-059 50 50 50 102 102 102 F F F 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 49 49 49 99 99 99 T T F <--- 8 160-179 50 50 50 102 102 102 T T F <--- 9 180-199 51 51 51 103 103 103 F F F 10 200-219 51 51 51 103 103 103 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 50 50 50 102 102 102 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 50 50 50 102 102 102 F F F 15 300-319 38 38 38 75 75 75 T T F <--- 16 320-339 49 49 49 99 99 99 T T F <--- 17 340-359 49 49 49 99 99 99 F F F 18 360-379 45 45 45 92 92 92 T T F <--- 19 380-399 51 51 51 103 103 103 F F F 20 400-419 50 50 50 102 102 102 T T F <--- 21 420-439 50 50 50 101 101 101 T T F <--- 22 440-459 52 52 49 106 106 99 F F T <--- 23 460-479 52 52 50 106 106 102 F F T <--- 24 480-499 50 50 50 102 102 102 F F F 25 500-519 52 52 52 104 104 104 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 52 52 52 105 105 105 F F F 28 560-579 50 50 50 102 102 102 T T F <--- 29 580-599 46 46 46 95 95 95 F F F Max gen: 52, Max flush: 106 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 26 Verifying data now No change after verify Loop: 26 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 50 50 50 102 102 102 F F F 2 040-059 50 50 50 102 102 102 F F F 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 47 47 47 96 96 96 F F F 7 140-159 53 53 53 107 107 107 F F F 8 160-179 53 53 53 107 107 107 F F F 9 180-199 51 51 51 103 103 103 F F F 10 200-219 51 51 51 103 103 103 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 50 50 50 102 102 102 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 50 50 50 102 102 102 F F F 15 300-319 53 53 53 107 107 107 F F F 16 320-339 53 53 53 107 107 107 F F F 17 340-359 49 49 49 99 99 99 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 51 51 51 103 103 103 F F F 20 400-419 53 53 53 107 107 107 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 52 52 52 106 106 106 F F F 23 460-479 52 52 52 106 106 106 F F F 24 480-499 50 50 50 102 102 102 F F F 25 500-519 52 52 52 104 104 104 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 52 52 52 105 105 105 F F F 28 560-579 53 53 53 107 107 107 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 53, Max flush: 107 Begin loop 27 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.90203069Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.913783046Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914146569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.91418866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914201325Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914219457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} {"msg":"Crucible 7d349d0d-a12a-4745-9b54-40219d5ce7a8 has session id: de07e9e5-14cc-4b2c-8e79-3fb2712ad677","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914543527Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Upstairs opts: Upstairs UUID: 7d349d0d-a12a-4745-9b54-40219d5ce7a8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914580241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Crucible stats registered with UUID: 7d349d0d-a12a-4745-9b54-40219d5ce7a8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914595104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:53.914608179Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914799736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914823865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914838669Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.914988164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"task":"crutest"} {"msg":"The guest has requested activation with gen:54","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915010644Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} {"msg":"7d349d0d-a12a-4745-9b54-40219d5ce7a8 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915074636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915090749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"0","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915104993Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"1","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915118897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"2","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915132471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915146775Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915160799Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"ds_connection connected from Ok(127.0.0.1:64444)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915185458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"0","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"ds_connection connected from Ok(127.0.0.1:35008)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915200542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"1","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"ds_connection connected from Ok(127.0.0.1:38208)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.915215425Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"io task","client":"2","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916025294Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"0","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916067036Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"2","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916083849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"1","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916388857Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916414176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[0]R flush_numbers[0..12]: [76, 102, 102, 102, 102, 96, 96, 107, 107, 103, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916477808Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[0]R generation[0..12]: [38, 50, 50, 50, 50, 47, 47, 53, 53, 51, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916495771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916515132Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[1]R flush_numbers[0..12]: [76, 102, 102, 102, 102, 96, 96, 107, 107, 103, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916531195Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[1]R generation[0..12]: [38, 50, 50, 50, 50, 47, 47, 53, 53, 51, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916546199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916562682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[2]R flush_numbers[0..12]: [76, 102, 102, 102, 102, 96, 96, 107, 107, 103, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916578265Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[2]R generation[0..12]: [38, 50, 50, 50, 50, 47, 47, 53, 53, 51, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916593238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916609681Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Max found gen is 54","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:50:53.916623145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Generation requested: 54 >= found:54","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916641457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Next flush: 108","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916654882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916668716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"":"downstairs","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.91668269Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916695474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"0","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916710258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"1","":"downstairs","session_idDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916730479Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"client":"2","":"downstairs","session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916745003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916758427Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"7d349d0d-a12a-4745-9b54-40219d5ce7a8 is now active with session: de07e9e5-14cc-4b2c-8e79-3fb2712ad677","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.9167738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916786704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416,"session_id":"de07e9e5-14cc-4b2c-8e79-3fb2712ad677"} {"msg":"The guest has finished waiting for activation with:54","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916806536Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} {"msg":"Activated sub_volume 7d349d0d-a12a-4745-9b54-40219d5ce7a8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:53.916818171Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 507 len 40960 data: 5 6 6 7 8 8 7 6 7 8 02/30 Flush 03/30 Write block 495 len 32768 data: 5 6 7 6 7 6 5 5 04/30 Read block 510 len 24576 05/30 Read block 469 len 8192 06/30 Read block 239 len 28672 07/30 Read block 150 len 24576 08/30 Read block 554 len 16384 09/30 Read block 479 len 12288 10/30 Flush 11/30 Write block 501 len 12288 data: 6 6 4 12/30 Read block 522 len 28672 13/30 Flush 14/30 Write block 136 len 24576 data: 7 7 7 7 7 8 15/30 Read block 232 len 8192 16/30 Flush 17/30 Write block 345 len 8192 data: 8 7 18/30 Read block 157 len 20480 19/30 Read block 67 len 20480 20/30 Write block 285 len 28672 data: 8 11 12 12 12 9 10 21/30 Read block 415 len 40960 22/30 Write block 556 len 28672 data: 4 3 3 3 3 3 2 23/30 Write block 47 len 8192 data: 5 5 24/30 Write block 191 len 36864 data: 6 6 8 9 7 9 10 9 9 25/30 Flush 26/30 Write block 187 len 4096 data: 5 27/30 Write block 157 len 20480 data: 2 3 4 6 6 28/30 Write block 494 len 24576 data: 4 6 7 8 7 8 29/30 Read block 331 len 8192 30/30 Write block 489 len 36864 data: 9 5 4 4 4 5 7 8 9 ---------------------------------------------------------------- Crucible gen:54 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 8 Done Done Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Read 2 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 4 Done Sent Done false 1008 Acked Read 3 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Read 7 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Write 2 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 1 Done Sent Done false 1026 Acked Write 5 Done Sent Done false 1027 Acked Write 6 Done Sent Done false 1028 Acked Read 2 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 27 1 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1001)) Some(JobId(1024)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:53.929399948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1416} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 50 50 50 102 102 102 F F F 2 040-059 54 50 54 112 102 112 F F F <--- 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 54 47 54 111 96 111 F F F <--- 7 140-159 54 53 54 111 107 111 T F T <--- 8 160-179 53 53 53 107 107 107 T F T <--- 9 180-199 54 51 54 112 103 112 T F T <--- 10 200-219 51 51 51 103 103 103 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 50 50 50 102 102 102 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 54 50 54 112 102 112 F F F <--- 15 300-319 53 53 53 107 107 107 F F F 16 320-339 53 53 53 107 107 107 F F F 17 340-359 54 49 54 112 99 112 F F F <--- 18 360-379 53 53 53 107 107 107 F F F 19 380-399 51 51 51 103 103 103 F F F 20 400-419 53 53 53 107 107 107 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 52 52 52 106 106 106 F F F 23 460-479 52 52 52 106 106 106 F F F 24 480-499 54 50 54 109 102 109 T T T <--- 25 500-519 54 54 54 110 108 110 F T F <--- 26 520-539 50 50 50 101 101 101 F F F 27 540-559 54 52 54 112 105 112 F F F <--- 28 560-579 54 53 54 112 107 112 F F F <--- 29 580-599 46 46 46 95 95 95 F F F Max gen: 54, Max flush: 112 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 27 Verifying data now No change after verify Loop: 27 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 50 50 50 102 102 102 F F F 2 040-059 54 54 54 112 112 112 F F F 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 54 54 54 111 111 111 F F F 7 140-159 55 55 55 113 113 113 F F F 8 160-179 55 55 55 113 113 113 F F F 9 180-199 55 55 55 113 113 113 F F F 10 200-219 51 51 51 103 103 103 F F F 11 220-239 38 38 38 73 73 73 F F F 12 240-259 50 50 50 102 102 102 F F F 13 260-279 36 36 36 70 70 70 F F F 14 280-299 54 54 54 112 112 112 F F F 15 300-319 53 53 53 107 107 107 F F F 16 320-339 53 53 53 107 107 107 F F F 17 340-359 54 54 54 112 112 112 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 51 51 51 103 103 103 F F F 20 400-419 53 53 53 107 107 107 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 52 52 52 106 106 106 F F F 23 460-479 52 52 52 106 106 106 F F F 24 480-499 55 55 55 113 113 113 F F F 25 500-519 54 54 54 110 110 110 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 54 54 54 112 112 112 F F F 28 560-579 54 54 54 112 112 112 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 55, Max flush: 113 Begin loop 28 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.248316113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.260054126Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.26036843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.260408932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.260421247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.260438829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} {"msg":"Crucible 51150eaf-b476-4924-8bb8-21b82615771d has session id: d515a454-d5d1-4f88-af3e-f6abc9cc7cf3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.260788857Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Upstairs opts: Upstairs UUID: 51150eaf-b476-4924-8bb8-21b82615771d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.260814176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Crucible stats registered with UUID: 51150eaf-b476-4924-8bb8-21b82615771d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.26082909Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:58.260844253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261065817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261103021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261118505Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261236333Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"task":"crutest"} {"msg":"The guest has requested activation with gen:56","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261253096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} {"msg":"51150eaf-b476-4924-8bb8-21b82615771d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261270399Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261283213Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"0","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261297397Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"1","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261311491Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"2","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261325415Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261339639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261354952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:49949)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261369826Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"1","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:46131)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261384689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"2","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:42529)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.261462435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"io task","client":"0","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262109445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"0","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262138452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"1","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262323922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"2","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262525365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262560759Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[0]R flush_numbers[0..12]: [76, 102, 112, 102, 102, 96, 111, 113, 113, 113, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262705377Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[0]R generation[0..12]: [38, 50, 54, 50, 50, 47, 54, 55, 55, 55, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262722979Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262739552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[1]R flush_numbers[0..12]: [76, 102, 112, 102, 102, 96, 111, 113, 113, 113, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262755625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[1]R generation[0..12]: [38, 50, 54, 50, 50, 47, 54, 55, 55, 55, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262771358Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262787901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[2]R flush_numbers[0..12]: [76, 102, 112, 102, 102, 96, 111, 113, 113, 113, 103, 73]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262803494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[2]R generation[0..12]: [38, 50, 54, 50, 50, 47, 54, 55, 55, 55, 51, 38]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262819197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.26283594Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Max found gen is 56","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262853253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424Wait for a query_work_queue command to finish before sending IO ,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Generation requested: 56 >= found:56","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262873994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Next flush: 114","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262896574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262909988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"":"downstairs","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262924062Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262936916Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"0","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.26295146Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"1","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262965094Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"client":"2","":"downstairs","session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262981867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.262995951Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"51150eaf-b476-4924-8bb8-21b82615771d is now active with session: d515a454-d5d1-4f88-af3e-f6abc9cc7cf3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.263011104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.263023999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424,"session_id":"d515a454-d5d1-4f88-af3e-f6abc9cc7cf3"} {"msg":"The guest has finished waiting for activation with:56","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.263037403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} {"msg":"Activated sub_volume 51150eaf-b476-4924-8bb8-21b82615771d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:50:58.263049098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 124 len 12288 data: 7 10 7 02/30 Write block 166 len 24576 data: 4 5 7 8 9 10 03/30 Read block 271 len 28672 04/30 Read block 448 len 24576 05/30 Write block 140 len 24576 data: 8 9 7 5 4 4 06/30 Flush 07/30 Write block 450 len 32768 data: 4 5 7 7 7 6 7 7 08/30 Write block 243 len 12288 data: 4 3 3 09/30 Read block 56 len 4096 10/30 Write block 177 len 4096 data: 4 11/30 Read block 121 len 24576 12/30 Read block 413 len 32768 13/30 Write block 492 len 20480 data: 5 5 6 8 9 14/30 Read block 94 len 28672 15/30 Read block 543 len 12288 16/30 Write block 349 len 24576 data: 8 8 7 8 6 5 17/30 Flush 18/30 Write block 298 len 16384 data: 6 6 6 7 19/30 Write block 169 len 32768 data: 9 10 11 9 6 6 6 5 20/30 Write block 203 len 20480 data: 7 8 8 6 5 21/30 Write block 479 len 40960 data: 5 7 7 9 8 7 6 6 6 8 22/30 Read block 218 len 24576 23/30 Read block 320 len 36864 24/30 Write block 27 len 32768 data: 4 5 7 6 6 6 7 7 25/30 Write block 223 len 32768 data: 4 4 3 2 2 3 4 4 26/30 Write block 558 len 8192 data: 4 4 27/30 Read block 352 len 4096 28/30 Write block 259 len 40960 data: 8 9 9 8 5 5 6 6 6 5 29/30 Read block 243 len 20480 30/30 Read block 127 len 8192 ---------------------------------------------------------------- Crucible gen:56 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Sent false 1001 Acked Write 6 Done Done Sent false 1002 Acked Read 7 Done Done Sent false 1003 Acked Read 6 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Flush 0 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Write 3 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 6 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Write 5 Done Done Sent false 1013 Acked Read 7 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 6 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 8 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Read 5 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1016)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:50:58.275079244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1424} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 50 50 50 102 102 102 T T F <--- 2 040-059 54 54 54 112 112 112 F F F 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 56 56 54 114 114 111 F F F <--- 7 140-159 56 56 55 114 114 113 F F F <--- 8 160-179 56 56 55 115 115 113 T T F <--- 9 180-199 55 55 55 113 113 113 F F F 10 200-219 51 51 51 103 103 103 T T F <--- 11 220-239 38 38 38 73 73 73 T T F <--- 12 240-259 56 56 50 115 115 102 T T F <--- 13 260-279 36 36 36 70 70 70 T T F <--- 14 280-299 54 54 54 112 112 112 T T F <--- 15 300-319 53 53 53 107 107 107 T T F <--- 16 320-339 53 53 53 107 107 107 F F F 17 340-359 56 56 54 115 115 112 F F F <--- 18 360-379 53 53 53 107 107 107 F F F 19 380-399 51 51 51 103 103 103 F F F 20 400-419 53 53 53 107 107 107 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 56 56 52 115 115 106 F F F <--- 23 460-479 52 52 52 106 106 106 T T F <--- 24 480-499 56 56 55 115 115 113 T T F <--- 25 500-519 54 54 54 110 110 110 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 54 54 54 112 112 112 T T F <--- 28 560-579 54 54 54 112 112 112 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 56, Max flush: 115 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 28 Verifying data now No change after verify Loop: 28 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 57 57 57 116 116 116 F F F 2 040-059 54 54 54 112 112 112 F F F 3 060-079 50 50 50 102 102 102 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F F F 6 120-139 56 56 56 114 114 114 F F F 7 140-159 56 56 56 114 114 114 F F F 8 160-179 57 57 57 116 116 116 F F F 9 180-199 55 55 55 113 113 113 F F F 10 200-219 57 57 57 116 116 116 F F F 11 220-239 57 57 57 116 116 116 F F F 12 240-259 57 57 57 116 116 116 F F F 13 260-279 57 57 57 116 116 116 F F F 14 280-299 57 57 57 116 116 116 F F F 15 300-319 57 57 57 116 116 116 F F F 16 320-339 53 53 53 107 107 107 F F F 17 340-359 56 56 56 115 115 115 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 51 51 51 103 103 103 F F F 20 400-419 53 53 53 107 107 107 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 56 56 56 115 115 115 F F F 23 460-479 57 57 57 116 116 116 F F F 24 480-499 57 57 57 116 116 116 F F F 25 500-519 54 54 54 110 110 110 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 57 57 57 116 116 116 F F F 28 560-579 54 54 54 112 112 112 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 57, Max flush: 116 Begin loop 29 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.606000165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:02.616715641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.616774496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.627327462Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.627629141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.627670723Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.627682918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.62770073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} {"msg":"Crucible e9619243-33c5-4b29-a87c-30be3b2511b0 has session id: 036195e7-78c6-4511-abd9-d0cd41a6a7a7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628074508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Upstairs opts: Upstairs UUID: e9619243-33c5-4b29-a87c-30be3b2511b0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628108144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Crucible stats registered with UUID: e9619243-33c5-4b29-a87c-30be3b2511b0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628123337Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:02.628136391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628311755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628338454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628352908Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.62850927Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"task":"crutest"} {"msg":"The guest has requested activation with gen:58","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628527282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} {"msg":"e9619243-33c5-4b29-a87c-30be3b2511b0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628542046Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.62855505Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"0","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628568994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"1","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628582748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"2","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628596262Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628610726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.62862485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:45599)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.62866988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"1","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:62078)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628799634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"2","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"ds_connection connected from Ok(127.0.0.1:55753)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.628833729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"io task","client":"0","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.629377114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"2","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.629406201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"1","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.629643868Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"0","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.629728252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.629763066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[0]R flush_numbers[0..12]: [76, 116, 112, 102, 102, 96, 114, 114, 116, 113, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630041536Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[0]R generation[0..12]: [38, 57, 54, 50, 50, 47, 56, 56, 57, 55, 57, 57]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630058878Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630075731Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[1]R flush_numbers[0..12]: [76, 116, 112, 102, 102, 96, 114, 114, 116, 113, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630091364Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[1]R generation[0..12]: [38, 57, 54, 50, 50, 47, 56, 56, 57, 55, 57, 57]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630106728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.63012292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[2]R flush_numbers[0..12]: [76, 116, 112, 102, 102, 96, 114, 114, 116, 113, 116, 116]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630138874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[2]R generation[0..12]: [38, 57, 54, 50, 50, 47, 56, 56, 57, 55, 57, 57]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630153997Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.63017035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Max found gen is 58","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:51:02.630184144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Generation requested: 58 >= found:58","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630202226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Next flush: 117","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.63021547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630228855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"":"downstairs","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630243008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630257552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"0","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630271716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"1","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630287469Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"client":"2","":"downstairs","session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630302093Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630315477Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"e9619243-33c5-4b29-a87c-30be3b2511b0 is now active with session: 036195e7-78c6-4511-abd9-d0cd41a6a7a7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.63033056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630343825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432,"session_id":"036195e7-78c6-4511-abd9-d0cd41a6a7a7"} {"msg":"The guest has finished waiting for activation with:58","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630357779Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} {"msg":"Activated sub_volume e9619243-33c5-4b29-a87c-30be3b2511b0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:02.630369294Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 50 len 28672 data: 7 8 7 7 8 8 7 02/30 Write block 234 len 12288 data: 5 5 5 03/30 Read block 365 len 8192 04/30 Flush 05/30 Write block 399 len 12288 data: 3 4 5 06/30 Read block 134 len 4096 07/30 Write block 319 len 12288 data: 6 5 4 08/30 Read block 133 len 12288 09/30 Read block 247 len 32768 10/30 Write block 391 len 32768 data: 5 6 6 4 2 3 3 3 11/30 Read block 204 len 8192 12/30 Read block 276 len 8192 13/30 Write block 147 len 12288 data: 4 7 7 14/30 Read block 149 len 36864 15/30 Read block 306 len 20480 16/30 Read block 33 len 28672 17/30 Read block 102 len 36864 18/30 Read block 58 len 32768 19/30 Write block 198 len 40960 data: 10 10 8 7 7 8 9 9 7 6 20/30 Write block 188 len 32768 data: 4 4 5 7 7 9 10 8 21/30 Flush 22/30 Write block 493 len 36864 data: 6 7 9 10 10 8 9 7 7 23/30 Write block 560 len 28672 data: 4 4 3 2 2 3 3 24/30 Flush 25/30 Write block 115 len 20480 data: 4 4 3 3 4 26/30 Write block 74 len 8192 data: 3 4 27/30 Read block 199 len 20480 28/30 Write block 392 len 36864 data: 7 7 5 3 4 4 4 4 5 29/30 Write block 130 len 28672 data: 4 3 4 5 6 7 8 30/30 Write block 293 len 16384 data: 6 6 6 6 ---------------------------------------------------------------- Crucible gen:58 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 3 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 9 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Write 10 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 7 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 5 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 9 Sent Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 3 2 32 Done 3 27 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:02.64263009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1432} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 57 57 57 116 116 116 F F F 2 040-059 54 58 58 112 117 117 T F F <--- 3 060-079 50 50 50 102 102 102 F T T <--- 4 080-099 50 50 50 102 102 102 F F F 5 100-119 47 47 47 96 96 96 F T T <--- 6 120-139 56 56 56 114 114 114 F T T <--- 7 140-159 56 58 58 114 118 118 F F F <--- 8 160-179 57 57 57 116 116 116 F F F 9 180-199 55 58 58 113 118 118 F F F <--- 10 200-219 57 58 58 116 118 118 F F F <--- 11 220-239 57 58 58 116 117 117 T F F <--- 12 240-259 57 57 57 116 116 116 F F F 13 260-279 57 57 57 116 116 116 F F F 14 280-299 57 57 57 116 116 116 F T T <--- 15 300-319 57 58 58 116 118 118 F F F <--- 16 320-339 53 58 58 107 118 118 F F F <--- 17 340-359 56 56 56 115 115 115 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 51 58 58 103 118 118 F T T <--- 20 400-419 53 58 58 107 118 118 F T T <--- 21 420-439 53 53 53 107 107 107 F F F 22 440-459 56 56 56 115 115 115 F F F 23 460-479 57 57 57 116 116 116 F F F 24 480-499 57 58 58 116 119 119 F F F <--- 25 500-519 54 58 58 110 119 119 F F F <--- 26 520-539 50 50 50 101 101 101 F F F 27 540-559 57 57 57 116 116 116 F F F 28 560-579 54 58 58 112 119 119 F F F <--- 29 580-599 46 46 46 95 95 95 F F F Max gen: 58, Max flush: 119 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 29 Verifying data now No change after verify Loop: 29 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 38 38 38 76 76 76 F F F 1 020-039 57 57 57 116 116 116 F F F 2 040-059 58 58 58 117 117 117 F F F 3 060-079 59 59 59 120 120 120 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 59 59 59 120 120 120 F F F 6 120-139 59 59 59 120 120 120 F F F 7 140-159 58 58 58 118 118 118 F F F 8 160-179 57 57 57 116 116 116 F F F 9 180-199 58 58 58 118 118 118 F F F 10 200-219 58 58 58 118 118 118 F F F 11 220-239 58 58 58 117 117 117 F F F 12 240-259 57 57 57 116 116 116 F F F 13 260-279 57 57 57 116 116 116 F F F 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 58 58 58 118 118 118 F F F 17 340-359 56 56 56 115 115 115 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 59 59 59 120 120 120 F F F 20 400-419 59 59 59 120 120 120 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 56 56 56 115 115 115 F F F 23 460-479 57 57 57 116 116 116 F F F 24 480-499 58 58 58 119 119 119 F F F 25 500-519 58 58 58 119 119 119 F F F 26 520-539 50 50 50 101 101 101 F F F 27 540-559 57 57 57 116 116 116 F F F 28 560-579 58 58 58 119 119 119 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 59, Max flush: 120 Begin loop 30 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.980259263Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992192841Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992512023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992549766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992562161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992580183Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} {"msg":"Crucible 69915428-b429-4b1b-91f4-fe79e1afbb75 has session id: 4a3c85a2-e5b0-40d5-b029-aad8725bc1b7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992937968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Upstairs opts: Upstairs UUID: 69915428-b429-4b1b-91f4-fe79e1afbb75, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992975202Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Crucible stats registered with UUID: 69915428-b429-4b1b-91f4-fe79e1afbb75","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.992990825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:06.993004289Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993192408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993217867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.99323256Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993375129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"task":"crutest"} {"msg":"The guest has requested activation with gen:60","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993392971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} {"msg":"69915428-b429-4b1b-91f4-fe79e1afbb75 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993409424Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993422048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"0","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993435862Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"1","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993449616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"2","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993463011Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993477114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993491318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:48484)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993589536Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"2","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:50332)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993661964Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"1","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41317)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.993678267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"io task","client":"0","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994311543Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"2","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994339561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"0","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994356513Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"1","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994702633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994746534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[0]R flush_numbers[0..12]: [76, 116, 117, 120, 102, 120, 120, 118, 116, 118, 118, 117]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994805329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[0]R generation[0..12]: [38, 57, 58, 59, 50, 59, 59, 58, 57, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994822181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994838914Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[1]R flush_numbers[0..12]: [76, 116, 117, 120, 102, 120, 120, 118, 116, 118, 118, 117]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994854877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[1]R generation[0..12]: [38, 57, 58, 59, 50, 59, 59, 58, 57, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994869891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994886254Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[2]R flush_numbers[0..12]: [76, 116, 117, 120, 102, 120, 120, 118, 116, 118, 118, 117]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994901937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[2]R generation[0..12]: [38, 57, 58, 59, 50, 59, 59, 58, 57, 58, 58, 58]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994919659Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994936112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Max found gen is 60","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:51:06.994949626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Generation requested: 60 >= found:60","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994968508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Next flush: 121","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994981952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.994995506Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"":"downstairs","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.9950095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995022575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"0","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995044005Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"1","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995058179Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"client":"2","":"downstairs","session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995075352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995089116Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"69915428-b429-4b1b-91f4-fe79e1afbb75 is now active with session: 4a3c85a2-e5b0-40d5-b029-aad8725bc1b7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995104289Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995116874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440,"session_id":"4a3c85a2-e5b0-40d5-b029-aad8725bc1b7"} {"msg":"The guest has finished waiting for activation with:60","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995130348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} {"msg":"Activated sub_volume 69915428-b429-4b1b-91f4-fe79e1afbb75","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:06.995143762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 19 len 24576 data: 2 3 3 3 5 6 02/30 Flush 03/30 Write block 155 len 40960 data: 4 4 3 4 5 7 7 7 5 4 04/30 Flush 05/30 Write block 217 len 32768 data: 3 2 2 3 4 4 5 5 06/30 Write block 354 len 20480 data: 6 6 7 8 8 07/30 Write block 527 len 16384 data: 2 2 3 3 08/30 Read block 380 len 24576 09/30 Write block 219 len 24576 data: 3 4 5 5 6 6 10/30 Read block 435 len 24576 11/30 Read block 574 len 16384 12/30 Write block 180 len 28672 data: 5 5 5 5 4 5 6 13/30 Read block 190 len 28672 14/30 Read block 173 len 24576 15/30 Read block 471 len 4096 16/30 Write block 326 len 12288 data: 4 5 4 17/30 Flush 18/30 Write block 42 len 12288 data: 4 4 5 19/30 Read block 168 len 4096 20/30 Write block 159 len 20480 data: 6 8 8 8 6 21/30 Read block 519 len 40960 22/30 Read block 533 len 40960 23/30 Read block 56 len 40960 24/30 Write block 128 len 40960 data: 5 5 5 4 5 6 7 8 9 8 25/30 Read block 126 len 36864 26/30 Flush 27/30 Write block 182 len 40960 data: 6 6 5 6 7 6 5 5 6 8 28/30 Write block 386 len 40960 data: 5 4 5 5 5 6 8 8 6 4 29/30 Read block 10 len 32768 30/30 Write block 495 len 32768 data: 10 11 11 9 10 8 8 7 ---------------------------------------------------------------- Crucible gen:60 GIO:true work queues: Upstairs:0 downstairs:28 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1002 Acked Write 10 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 8 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Write 6 Done Sent Done false 1009 Acked Read 6 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Read 1 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 10 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 10 Done Sent Done false 1024 Acked Read 9 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 10 Done Sent Done false 1027 Acked Write 10 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 27 0 27 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1001)) Some(JobId(1025)) Downstairs last five completed: 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:07.009486416Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1440} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 58 60 124 117 124 F F F <--- 3 060-079 59 59 59 120 120 120 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 59 59 59 120 120 120 F F F 6 120-139 60 59 60 124 120 124 F F F <--- 7 140-159 60 58 60 124 118 124 F F F <--- 8 160-179 60 57 60 124 116 124 F F F <--- 9 180-199 60 58 60 123 118 123 T F T <--- 10 200-219 60 58 60 123 118 123 F F F <--- 11 220-239 60 58 60 123 117 123 F F F <--- 12 240-259 57 57 57 116 116 116 F F F 13 260-279 57 57 57 116 116 116 F F F 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 60 58 60 123 118 123 F F F <--- 17 340-359 60 56 60 123 115 123 F F F <--- 18 360-379 53 53 53 107 107 107 F F F 19 380-399 59 59 59 120 120 120 T F T <--- 20 400-419 59 59 59 120 120 120 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 56 56 56 115 115 115 F F F 23 460-479 57 57 57 116 116 116 F F F 24 480-499 58 58 58 119 119 119 T F T <--- 25 500-519 58 58 58 119 119 119 T F T <--- 26 520-539 60 50 60 123 101 123 F F F <--- 27 540-559 57 57 57 116 116 116 F F F 28 560-579 58 58 58 119 119 119 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 60, Max flush: 124 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 30 Verifying data now No change after verify Loop: 30 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 59 59 59 120 120 120 F F F 4 080-099 50 50 50 102 102 102 F F F 5 100-119 59 59 59 120 120 120 F F F 6 120-139 60 60 60 124 124 124 F F F 7 140-159 60 60 60 124 124 124 F F F 8 160-179 60 60 60 124 124 124 F F F 9 180-199 61 61 61 125 125 125 F F F 10 200-219 60 60 60 123 123 123 F F F 11 220-239 60 60 60 123 123 123 F F F 12 240-259 57 57 57 116 116 116 F F F 13 260-279 57 57 57 116 116 116 F F F 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 60 60 60 123 123 123 F F F 17 340-359 60 60 60 123 123 123 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 61 61 61 125 125 125 F F F 20 400-419 59 59 59 120 120 120 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 56 56 56 115 115 115 F F F 23 460-479 57 57 57 116 116 116 F F F 24 480-499 61 61 61 125 125 125 F F F 25 500-519 61 61 61 125 125 125 F F F 26 520-539 60 60 60 123 123 123 F F F 27 540-559 57 57 57 116 116 116 F F F 28 560-579 58 58 58 119 119 119 F F F 29 580-599 46 46 46 95 95 95 F F F Max gen: 61, Max flush: 125 Begin loop 31 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.338958014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:11.349705637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.349758943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.360174999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.36047197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.360512673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.360524598Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.36054236Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} {"msg":"Crucible 6ba9095d-af4a-4d9b-afdd-6bbd1b76944e has session id: 438c6f9e-63d1-4a76-b50c-e15c9d8433b0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.360905643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Upstairs opts: Upstairs UUID: 6ba9095d-af4a-4d9b-afdd-6bbd1b76944e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.360938888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Crucible stats registered with UUID: 6ba9095d-af4a-4d9b-afdd-6bbd1b76944e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.360953672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:11.360967176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361174556Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361200115Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361217567Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361337795Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"task":"crutest"} {"msg":"The guest has requested activation with gen:62","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361355568Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} {"msg":"6ba9095d-af4a-4d9b-afdd-6bbd1b76944e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.36137288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361385485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"0","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361399359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"1","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361413013Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"2","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361426397Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.36144175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361455844Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:46948)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361504023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"1","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:65200)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361520156Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"2","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"ds_connection connected from Ok(127.0.0.1:65507)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.361682286Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"io task","client":"0","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362178201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"1","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362214865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"2","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362417737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"0","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362546651Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362585444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[0]R flush_numbers[0..12]: [121, 121, 124, 120, 102, 120, 124, 124, 124, 125, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362800141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[0]R generation[0..12]: [60, 60, 60, 59, 50, 59, 60, 60, 60, 61, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362819133Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362835966Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[1]R flush_numbers[0..12]: [121, 121, 124, 120, 102, 120, 124, 124, 124, 125, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362851799Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[1]R generation[0..12]: [60, 60, 60, 59, 50, 59, 60, 60, 60, 61, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362867392Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362883625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[2]R flush_numbers[0..12]: [121, 121, 124, 120, 102, 120, 124, 124, 124, 125, 123, 123]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362899239Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[2]R generation[0..12]: [60, 60, 60, 59, 50, 59, 60, 60, 60, 61, 60, 60]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362914602Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362931115Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Max found gen is 62","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362944719Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Generation requested: 62 >= found:62","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.3629642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Next flush: 126","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362978024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.362993188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"":"downstairs","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363007272Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363020066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"0","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.36303375Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"1","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363047854Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"client":"2","":"downstairs","session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:51:11.363062608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363078921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"6ba9095d-af4a-4d9b-afdd-6bbd1b76944e is now active with session: 438c6f9e-63d1-4a76-b50c-e15c9d8433b0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363093834Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363106649Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448,"session_id":"438c6f9e-63d1-4a76-b50c-e15c9d8433b0"} {"msg":"The guest has finished waiting for activation with:62","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363119803Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} {"msg":"Activated sub_volume 6ba9095d-af4a-4d9b-afdd-6bbd1b76944e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:11.363131318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 455 len 32768 data: 7 8 8 5 5 5 6 4 03/30 Read block 83 len 16384 04/30 Flush 05/30 Write block 335 len 16384 data: 5 6 6 7 06/30 Write block 468 len 32768 data: 8 8 7 8 5 5 5 4 07/30 Write block 481 len 24576 data: 8 10 9 8 7 7 08/30 Read block 459 len 36864 09/30 Write block 527 len 20480 data: 3 3 4 4 4 10/30 Flush 11/30 Write block 582 len 16384 data: 7 6 6 7 12/30 Write block 223 len 4096 data: 7 13/30 Read block 11 len 32768 14/30 Read block 173 len 12288 15/30 Read block 470 len 20480 16/30 Read block 77 len 4096 17/30 Read block 490 len 32768 18/30 Read block 514 len 40960 19/30 Write block 563 len 32768 data: 3 3 4 4 3 4 5 4 20/30 Read block 410 len 36864 21/30 Read block 222 len 16384 22/30 Write block 350 len 32768 data: 9 8 9 7 7 7 8 9 23/30 Read block 172 len 8192 24/30 Read block 482 len 4096 25/30 Write block 64 len 28672 data: 6 5 5 5 5 4 4 26/30 Write block 273 len 8192 data: 6 4 27/30 Write block 78 len 32768 data: 5 5 5 5 6 6 7 8 28/30 Write block 116 len 4096 data: 5 29/30 Write block 160 len 8192 data: 9 9 30/30 Write block 139 len 4096 data: 8 ---------------------------------------------------------------- Crucible gen:62 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 4 Sent Done Done false 1005 Acked Write 8 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 9 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 4 Sent Done Done false 1011 Acked Write 1 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Read 3 Sent Done Done false 1014 Acked Read 5 Sent Done Done false 1015 Acked Read 1 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 4 Sent Done Done false 1021 Acked Write 8 Sent Done Done false 1022 Acked Read 2 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 8 Sent Sent Sent false 1027 Acked Write 1 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 1 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 4 4 34 Done 0 22 22 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1009)) Some(JobId(1009)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:11.374497722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1448} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 59 59 59 120 120 120 F T T <--- 4 080-099 50 50 50 102 102 102 F T T <--- 5 100-119 59 59 59 120 120 120 F T T <--- 6 120-139 60 60 60 124 124 124 F T T <--- 7 140-159 60 60 60 124 124 124 F F F 8 160-179 60 60 60 124 124 124 F T T <--- 9 180-199 61 61 61 125 125 125 F F F 10 200-219 60 60 60 123 123 123 F F F 11 220-239 60 60 60 123 123 123 F T T <--- 12 240-259 57 57 57 116 116 116 F F F 13 260-279 57 57 57 116 116 116 F T T <--- 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 60 62 62 123 128 128 F F F <--- 17 340-359 60 60 60 123 123 123 F T T <--- 18 360-379 53 53 53 107 107 107 F F F 19 380-399 61 61 61 125 125 125 F F F 20 400-419 59 59 59 120 120 120 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 62 62 62 127 127 127 F F F 23 460-479 62 62 62 127 128 128 F F F <--- 24 480-499 61 62 62 125 128 128 F F F <--- 25 500-519 61 61 61 125 125 125 F F F 26 520-539 60 62 62 123 128 128 F F F <--- 27 540-559 57 57 57 116 116 116 F F F 28 560-579 58 58 58 119 119 119 F T T <--- 29 580-599 46 46 46 95 95 95 F T T <--- Max gen: 62, Max flush: 128 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 31 Verifying data now No change after verify Loop: 31 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 63 63 63 129 129 129 F F F 4 080-099 63 63 63 129 129 129 F F F 5 100-119 63 63 63 129 129 129 F F F 6 120-139 63 63 63 129 129 129 F F F 7 140-159 60 60 60 124 124 124 F F F 8 160-179 63 63 63 129 129 129 F F F 9 180-199 61 61 61 125 125 125 F F F 10 200-219 60 60 60 123 123 123 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 57 57 57 116 116 116 F F F 13 260-279 63 63 63 129 129 129 F F F 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 62 62 62 128 128 128 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 53 53 53 107 107 107 F F F 19 380-399 61 61 61 125 125 125 F F F 20 400-419 59 59 59 120 120 120 F F F 21 420-439 53 53 53 107 107 107 F F F 22 440-459 62 62 62 127 127 127 F F F 23 460-479 62 62 62 128 128 128 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 61 61 61 125 125 125 F F F 26 520-539 62 62 62 128 128 128 F F F 27 540-559 57 57 57 116 116 116 F F F 28 560-579 63 63 63 129 129 129 F F F 29 580-599 63 63 63 129 129 129 F F F Max gen: 63, Max flush: 129 Begin loop 32 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.730972468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743120223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743464784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743502637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743515372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743533174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} {"msg":"Crucible b453fa43-92f0-461e-8419-bd7e84395110 has session id: 98be3a2b-2edc-40a4-b05d-1fc83c4754c2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743907872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Upstairs opts: Upstairs UUID: b453fa43-92f0-461e-8419-bd7e84395110, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.743942527Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Crucible stats registered with UUID: b453fa43-92f0-461e-8419-bd7e84395110","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.74395832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:15.743972634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744153175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.74418852Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744203893Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744318084Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"task":"crutest"} {"msg":"The guest has requested activation with gen:64","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744335147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} {"msg":"b453fa43-92f0-461e-8419-bd7e84395110 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.74435018Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744362795Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"0","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744376519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"1","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744390213Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"2","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744404327Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.74441834Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744435063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:58039)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744449937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"0","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:56095)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.74446454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"1","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:47044)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.744498376Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"io task","client":"2","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745247521Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"0","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745270291Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"1","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745286324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"2","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745614472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745629785Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[0]R flush_numbers[0..12]: [121, 121, 124, 129, 129, 129, 129, 124, 129, 125, 123, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745663811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[0]R generation[0..12]: [60, 60, 60, 63, 63, 63, 63, 60, 63, 61, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745680533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745697096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[1]R flush_numbers[0..12]: [121, 121, 124, 129, 129, 129, 129, 124, 129, 125, 123, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745712699Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[1]R generation[0..12]: [60, 60, 60, 63, 63, 63, 63, 60, 63, 61, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745728053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745746705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[2]R flush_numbers[0..12]: [121, 121, 124, 129, 129, 129, 129, 124, 129, 125, 123, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745762508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[2]R generation[0..12]: [60, 60, 60, 63, 63, 63, 63, 60, 63, 61, 60, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745777591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745793784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Max found gen is 64","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745806909Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","Wait for a query_work_queue command to finish before sending IO session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Generation requested: 64 >= found:64","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.7458276Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Next flush: 130","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745841344Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745854618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"":"downstairs","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745868702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745881166Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"0","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745894911Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"1","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745908585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"client":"2","":"downstairs","session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:51:15.745922639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745938202Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"b453fa43-92f0-461e-8419-bd7e84395110 is now active with session: 98be3a2b-2edc-40a4-b05d-1fc83c4754c2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745953245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745967569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456,"session_id":"98be3a2b-2edc-40a4-b05d-1fc83c4754c2"} {"msg":"The guest has finished waiting for activation with:64","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745980943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} {"msg":"Activated sub_volume b453fa43-92f0-461e-8419-bd7e84395110","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:15.745992418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 364 len 36864 data: 6 4 4 5 4 5 5 3 3 02/30 Write block 266 len 28672 data: 7 7 6 6 4 4 5 03/30 Read block 85 len 20480 04/30 Read block 584 len 32768 05/30 Write block 544 len 36864 data: 2 2 2 5 5 5 5 5 4 06/30 Write block 415 len 40960 data: 5 5 6 6 7 6 5 6 6 6 07/30 Write block 529 len 12288 data: 5 5 5 08/30 Write block 100 len 20480 data: 6 5 4 4 5 09/30 Read block 430 len 28672 10/30 Write block 422 len 16384 data: 7 7 7 6 11/30 Write block 565 len 32768 data: 5 5 4 5 6 5 3 3 12/30 Read block 159 len 32768 13/30 Flush 14/30 Write block 588 len 24576 data: 5 4 4 4 5 4 15/30 Read block 237 len 8192 16/30 Read block 590 len 36864 17/30 Read block 207 len 20480 18/30 Read block 401 len 36864 19/30 Write block 380 len 16384 data: 6 7 6 5 20/30 Read block 283 len 4096 21/30 Read block 8 len 36864 22/30 Write block 208 len 32768 data: 5 6 4 5 5 4 4 3 23/30 Read block 331 len 36864 24/30 Write block 151 len 24576 data: 8 5 4 4 5 5 25/30 Write block 251 len 28672 data: 4 4 5 5 5 5 8 26/30 Read block 221 len 16384 27/30 Write block 325 len 28672 data: 3 5 6 5 4 5 5 28/30 Write block 191 len 40960 data: 9 8 10 11 9 10 11 11 11 9 29/30 Write block 168 len 4096 data: 8 30/30 Read block 3 len 16384 ---------------------------------------------------------------- Crucible gen:64 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 8 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Write 10 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 7 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 8 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Write 4 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Read 9 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Read 4 Done Done Sent false 1026 Acked Write 7 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:15.758938188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1456} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 63 63 63 129 129 129 F F F 4 080-099 63 63 63 129 129 129 F F F 5 100-119 64 64 63 130 130 129 F F F <--- 6 120-139 63 63 63 129 129 129 F F F 7 140-159 60 60 60 124 124 124 T T F <--- 8 160-179 63 63 63 129 129 129 T T F <--- 9 180-199 61 61 61 125 125 125 T T F <--- 10 200-219 60 60 60 123 123 123 T T F <--- 11 220-239 63 63 63 129 129 129 F F F 12 240-259 57 57 57 116 116 116 T T F <--- 13 260-279 64 64 63 130 130 129 F F T <--- 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 62 62 62 128 128 128 T T F <--- 17 340-359 63 63 63 129 129 129 F F F 18 360-379 64 64 53 130 130 107 F F T <--- 19 380-399 61 61 61 125 125 125 T T F <--- 20 400-419 64 64 59 130 130 120 F F F <--- 21 420-439 64 64 53 130 130 107 F F F <--- 22 440-459 62 62 62 127 127 127 F F F 23 460-479 62 62 62 128 128 128 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 61 61 61 125 125 125 F F F 26 520-539 64 64 62 130 130 128 F F F <--- 27 540-559 64 64 57 130 130 116 F F F <--- 28 560-579 64 64 63 130 130 129 F F F <--- 29 580-599 63 63 63 129 129 129 T T F <--- Max gen: 64, Max flush: 130 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 32 Verifying data now No change after verify Loop: 32 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 63 63 63 129 129 129 F F F 4 080-099 63 63 63 129 129 129 F F F 5 100-119 64 64 64 130 130 130 F F F 6 120-139 63 63 63 129 129 129 F F F 7 140-159 65 65 65 131 131 131 F F F 8 160-179 65 65 65 131 131 131 F F F 9 180-199 65 65 65 131 131 131 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 65 65 65 131 131 131 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 59 59 59 120 120 120 F F F 15 300-319 58 58 58 118 118 118 F F F 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 64 64 64 130 130 130 F F F 19 380-399 65 65 65 131 131 131 F F F 20 400-419 64 64 64 130 130 130 F F F 21 420-439 64 64 64 130 130 130 F F F 22 440-459 62 62 62 127 127 127 F F F 23 460-479 62 62 62 128 128 128 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 61 61 61 125 125 125 F F F 26 520-539 64 64 64 130 130 130 F F F 27 540-559 64 64 64 130 130 130 F F F 28 560-579 64 64 64 130 130 130 F F F 29 580-599 65 65 65 131 131 131 F F F Max gen: 65, Max flush: 131 Begin loop 33 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.093070916Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.104853319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105179178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.10522003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105232625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105250447Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} {"msg":"Crucible 54fe9ce0-ff45-4b20-a90e-d84046a77eab has session id: cb31dbe0-b126-4599-8e48-3c78d4dab478","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.1056125Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Upstairs opts: Upstairs UUID: 54fe9ce0-ff45-4b20-a90e-d84046a77eab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105696524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Crucible stats registered with UUID: 54fe9ce0-ff45-4b20-a90e-d84046a77eab","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105714866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:20.10572859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105910291Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105940518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.105955432Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106066564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"task":"crutest"} {"msg":"The guest has requested activation with gen:66","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106083586Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} {"msg":"54fe9ce0-ff45-4b20-a90e-d84046a77eab active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106102958Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106116052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"0","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106130036Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"1","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"2","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106157554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106171988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106186422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"ds_connection connected from Ok(127.0.0.1:38374)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106222476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"2","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"ds_connection connected from Ok(127.0.0.1:64569)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106312727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"0","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"ds_connection connected from Ok(127.0.0.1:59382)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106423899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"io task","client":"1","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106911577Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"0","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.106939065Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"1","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107149294Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"2","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107291203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107330006Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[0]R flush_numbers[0..12]: [121, 121, 124, 129, 129, 130, 129, 131, 131, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.10752816Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[0]R generation[0..12]: [60, 60, 60, 63, 63, 64, 63, 65, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107545293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107561776Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[1]R flush_numbers[0..12]: [121, 121, 124, 129, 129, 130, 129, 131, 131, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107577879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[1]R generation[0..12]: [60, 60, 60, 63, 63, 64, 63, 65, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107593252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107609735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[2]R flush_numbers[0..12]: [121, 121, 124, 129, 129, 130, 129, 131, 131, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107625398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[2]R generation[0..12]: [60, 60, 60, 63, 63, 64, 63, 65, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107640451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107656704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Max found gen is 66","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107670129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 66 >= found:66","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.10769004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Next flush: 132","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107705943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107719567Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"":"downstairs","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107733751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107746575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"0","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107760649Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"1","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107774743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"client":"2","":"downstairs","session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107791846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.10780543Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"54fe9ce0-ff45-4b20-a90e-d84046a77eab is now active with session: cb31dbe0-b126-4599-8e48-3c78d4dab478","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107820913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107833548Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464,"session_id":"cb31dbe0-b126-4599-8e48-3c78d4dab478"} {"msg":"The guest has finished waiting for activation with:66","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107847002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} {"msg":"Activated sub_volume 54fe9ce0-ff45-4b20-a90e-d84046a77eab","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:20.107858557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 511 len 16384 data: 9 9 8 7 02/30 Read block 277 len 28672 03/30 Read block 142 len 4096 04/30 Write block 525 len 36864 data: 4 3 4 4 6 6 6 5 5 05/30 Write block 545 len 40960 data: 3 3 6 6 6 6 6 5 5 5 06/30 Write block 386 len 32768 data: 6 5 6 6 6 7 9 9 07/30 Read block 429 len 16384 08/30 Read block 464 len 28672 09/30 Flush 10/30 Write block 97 len 36864 data: 7 6 6 7 6 5 5 6 4 11/30 Read block 208 len 20480 12/30 Write block 284 len 40960 data: 7 9 12 13 13 13 10 11 7 7 13/30 Read block 307 len 40960 14/30 Read block 63 len 8192 15/30 Read block 52 len 32768 16/30 Read block 87 len 12288 17/30 Write block 290 len 40960 data: 11 12 8 8 7 7 7 8 7 7 18/30 Write block 285 len 12288 data: 10 13 14 19/30 Read block 377 len 16384 20/30 Write block 531 len 20480 data: 7 6 6 4 3 21/30 Read block 546 len 36864 22/30 Flush 23/30 Write block 415 len 4096 data: 6 24/30 Read block 562 len 36864 25/30 Flush 26/30 Write block 312 len 4096 data: 5 27/30 Write block 142 len 16384 data: 8 6 5 5 28/30 Read block 219 len 4096 29/30 Read block 17 len 12288 30/30 Write block 518 len 32768 data: 7 7 7 5 4 4 4 5 ---------------------------------------------------------------- Crucible gen:66 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Sent false 1001 Acked Read 7 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Write 9 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Read 4 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 10 Done Done Sent false 1012 Acked Read 10 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Write 10 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 1 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 1 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 30 32 Done 29 29 0 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:20.120624524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1464} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 63 63 63 129 129 129 F F F 4 080-099 66 66 63 133 133 129 F F F <--- 5 100-119 66 66 64 133 133 130 F F F <--- 6 120-139 63 63 63 129 129 129 F F F 7 140-159 65 65 65 131 131 131 T T F <--- 8 160-179 65 65 65 131 131 131 F F F 9 180-199 65 65 65 131 131 131 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 65 65 65 131 131 131 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 59 133 133 120 F F F <--- 15 300-319 58 58 58 118 118 118 T T F <--- 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 64 64 64 130 130 130 F F F 19 380-399 66 66 65 132 132 131 F F F <--- 20 400-419 66 66 64 134 134 130 F F F <--- 21 420-439 64 64 64 130 130 130 F F F 22 440-459 62 62 62 127 127 127 F F F 23 460-479 62 62 62 128 128 128 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 66 66 61 132 132 125 T T F <--- 26 520-539 66 66 64 133 133 130 T T F <--- 27 540-559 66 66 64 132 132 130 F F F <--- 28 560-579 64 64 64 130 130 130 F F F 29 580-599 65 65 65 131 131 131 F F F Max gen: 66, Max flush: 134 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 33 Verifying data now No change after verify Loop: 33 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 F F F 1 020-039 60 60 60 121 121 121 F F F 2 040-059 60 60 60 124 124 124 F F F 3 060-079 63 63 63 129 129 129 F F F 4 080-099 66 66 66 133 133 133 F F F 5 100-119 66 66 66 133 133 133 F F F 6 120-139 63 63 63 129 129 129 F F F 7 140-159 67 67 67 135 135 135 F F F 8 160-179 65 65 65 131 131 131 F F F 9 180-199 65 65 65 131 131 131 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 65 65 65 131 131 131 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 67 67 67 135 135 135 F F F 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 64 64 64 130 130 130 F F F 19 380-399 66 66 66 132 132 132 F F F 20 400-419 66 66 66 134 134 134 F F F 21 420-439 64 64 64 130 130 130 F F F 22 440-459 62 62 62 127 127 127 F F F 23 460-479 62 62 62 128 128 128 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 67 67 67 135 135 135 F F F 26 520-539 67 67 67 135 135 135 F F F 27 540-559 66 66 66 132 132 132 F F F 28 560-579 64 64 64 130 130 130 F F F 29 580-599 65 65 65 131 131 131 F F F Max gen: 67, Max flush: 135 Begin loop 34 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.435467014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447152639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447481766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447523149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447538232Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447556054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} {"msg":"Crucible bdfafb0e-cd02-4440-b67f-3562e0b0918a has session id: 0eaf04dd-1a18-45cf-acd6-e9475bc4dabc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447902904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Upstairs opts: Upstairs UUID: bdfafb0e-cd02-4440-b67f-3562e0b0918a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.447943196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Crucible stats registered with UUID: bdfafb0e-cd02-4440-b67f-3562e0b0918a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.44795883Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:24.447972634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448160302Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448196637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.44821231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448335826Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"task":"crutest"} {"msg":"The guest has requested activation with gen:68","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448359866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} {"msg":"bdfafb0e-cd02-4440-b67f-3562e0b0918a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.44837242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448385555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"0","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448400058Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"1","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448414342Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"2","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448430785Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448445219Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448471577Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"ds_connection connected from Ok(127.0.0.1:65127)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448486831Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"1","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"ds_connection connected from Ok(127.0.0.1:34149)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448501864Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"2","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"ds_connection connected from Ok(127.0.0.1:35991)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.448516698Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"io task","client":"0","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.449264954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"0","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.449292652Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"2","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.449621399Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.449661612Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"1","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.44968881Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[0]R flush_numbers[0..12]: [121, 121, 124, 129, 133, 133, 129, 135, 131, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450067856Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[0]R generation[0..12]: [60, 60, 60, 63, 66, 66, 63, 67, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450084718Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450101401Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[1]R flush_numbers[0..12]: [121, 121, 124, 129, 133, 133, 129, 135, 131, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450120293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[1]R generation[0..12]: [60, 60, 60, 63, 66, 66, 63, 67, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450135856Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450151989Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[2]R flush_numbers[0..12]: [121, 121, 124, 129, 133, 133, 129, 135, 131, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450167782Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[2]R generation[0..12]: [60, 60, 60, 63, 66, 66, 63, 67, 65, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450183236Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450199918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Max found gen is 68","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:51:24.450213672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Generation requested: 68 >= found:68","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450233954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Next flush: 136","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450247478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450260902Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"":"downstairs","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450275216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.45028797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"0","":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450304983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"1","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450319127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"client":"2","":"downstairs","session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450333481Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450348634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"bdfafb0e-cd02-4440-b67f-3562e0b0918a is now active with session: 0eaf04dd-1a18-45cf-acd6-e9475bc4dabc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450363757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450376372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472,"session_id":"0eaf04dd-1a18-45cf-acd6-e9475bc4dabc"} {"msg":"The guest has finished waiting for activation with:68","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450389616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} {"msg":"Activated sub_volume bdfafb0e-cd02-4440-b67f-3562e0b0918a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:24.450401171Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 374 len 24576 data: 3 3 4 3 4 4 02/30 Write block 409 len 12288 data: 9 7 8 03/30 Write block 36 len 28672 data: 8 5 5 6 7 6 5 04/30 Flush 05/30 Write block 389 len 8192 data: 7 7 06/30 Write block 115 len 32768 data: 5 6 4 4 5 6 6 7 07/30 Write block 80 len 4096 data: 6 08/30 Write block 100 len 12288 data: 8 7 6 09/30 Write block 21 len 32768 data: 4 4 6 7 5 5 5 6 10/30 Flush 11/30 Write block 308 len 36864 data: 5 6 5 5 6 4 4 5 5 12/30 Read block 17 len 12288 13/30 Write block 154 len 36864 data: 5 6 6 4 5 7 10 10 9 14/30 Read block 590 len 16384 15/30 Read block 180 len 16384 16/30 Write block 459 len 24576 data: 6 6 7 5 5 6 17/30 Write block 535 len 8192 data: 4 3 18/30 Write block 95 len 28672 data: 8 7 8 7 7 9 8 19/30 Read block 17 len 32768 20/30 Read block 358 len 4096 21/30 Write block 572 len 24576 data: 4 2 3 3 4 4 22/30 Read block 7 len 28672 23/30 Write block 142 len 28672 data: 9 7 6 6 4 5 8 24/30 Write block 512 len 28672 data: 10 9 8 8 9 9 8 25/30 Read block 579 len 8192 26/30 Read block 332 len 12288 27/30 Write block 73 len 32768 data: 2 4 5 4 4 6 6 7 28/30 Write block 587 len 36864 data: 7 6 5 5 5 6 5 4 3 29/30 Write block 5 len 24576 data: 3 4 4 5 5 5 30/30 Write block 116 len 32768 data: 7 5 5 6 7 7 8 8 ---------------------------------------------------------------- Crucible gen:68 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 3 Done Sent Done false 1002 Acked Write 7 Done Sent Done false 1003 Acked Flush 0 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 9 Done Sent Done false 1011 Acked Read 3 Done Sent Done false 1012 Acked Write 9 Done Sent Done false 1013 Acked Read 4 Done Sent Done false 1014 Acked Read 4 Done Sent Done false 1015 Acked Write 6 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 1 Done Sent Done false 1020 Acked Write 6 Done Sent Done false 1021 Acked Read 7 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Read 2 Done Sent Done false 1025 Acked Read 3 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Write 9 Done Sent Done false 1028 Acked Write 6 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 29 2 33 Done 28 1 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) None Some(JobId(1009)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:24.462347974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1472} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 60 60 60 121 121 121 T F T <--- 1 020-039 68 60 68 137 121 137 F F F <--- 2 040-059 68 60 68 136 124 136 F F F <--- 3 060-079 63 63 63 129 129 129 T F T <--- 4 080-099 68 66 68 137 133 137 T F T <--- 5 100-119 68 66 68 137 133 137 T F T <--- 6 120-139 68 63 68 137 129 137 T F T <--- 7 140-159 67 67 67 135 135 135 T F T <--- 8 160-179 65 65 65 131 131 131 T F T <--- 9 180-199 65 65 65 131 131 131 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 65 65 65 131 131 131 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 67 67 67 135 135 135 T F T <--- 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 68 64 68 136 130 136 F T F <--- 19 380-399 68 66 68 137 132 137 F F F <--- 20 400-419 68 66 68 136 134 136 F F F <--- 21 420-439 64 64 64 130 130 130 F F F 22 440-459 62 62 62 127 127 127 T F T <--- 23 460-479 62 62 62 128 128 128 T F T <--- 24 480-499 62 62 62 128 128 128 F F F 25 500-519 67 67 67 135 135 135 T F T <--- 26 520-539 67 67 67 135 135 135 T F T <--- 27 540-559 66 66 66 132 132 132 F F F 28 560-579 64 64 64 130 130 130 T F T <--- 29 580-599 65 65 65 131 131 131 T F T <--- Max gen: 68, Max flush: 137 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 34 Verifying data now No change after verify Loop: 34 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 138 138 138 F F F 1 020-039 68 68 68 137 137 137 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 69 69 69 138 138 138 F F F 4 080-099 69 69 69 138 138 138 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 69 69 69 138 138 138 F F F 7 140-159 69 69 69 138 138 138 F F F 8 160-179 69 69 69 138 138 138 F F F 9 180-199 65 65 65 131 131 131 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 65 65 65 131 131 131 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 69 69 69 138 138 138 F F F 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 68 68 68 136 136 136 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 68 68 68 136 136 136 F F F 21 420-439 64 64 64 130 130 130 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 69 69 69 138 138 138 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 66 66 66 132 132 132 F F F 28 560-579 69 69 69 138 138 138 F F F 29 580-599 69 69 69 138 138 138 F F F Max gen: 69, Max flush: 138 Begin loop 35 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.804864119Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.816609689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.816962686Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817003658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817015993Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817033915Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} {"msg":"Crucible 358c38a3-d3a6-4d2d-b5fa-974f23b216ed has session id: 15aaf551-b1db-45f0-94dc-94d92fd20707","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817350738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Upstairs opts: Upstairs UUID: 358c38a3-d3a6-4d2d-b5fa-974f23b216ed, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817396218Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Crucible stats registered with UUID: 358c38a3-d3a6-4d2d-b5fa-974f23b216ed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817416139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:28.817439209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817627938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817665701Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817681464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817808609Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"task":"crutest"} {"msg":"The guest has requested activation with gen:70","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817825742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} {"msg":"358c38a3-d3a6-4d2d-b5fa-974f23b216ed active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817847742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817861476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"0","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.81787563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"1","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817890504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"2","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817904318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817918442Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.817932955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"ds_connection connected from Ok(127.0.0.1:38810)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.818039659Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"0","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"ds_connection connected from Ok(127.0.0.1:39573)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.818192703Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"1","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"ds_connection connected from Ok(127.0.0.1:54550)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.818220721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"io task","client":"2","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.818768823Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"0","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.818797261Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"2","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.81882191Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"1","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.81914452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819190351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[0]R flush_numbers[0..12]: [138, 137, 136, 138, 138, 138, 138, 138, 138, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819247116Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[0]R generation[0..12]: [69, 68, 68, 69, 69, 69, 69, 69, 69, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819263939Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819280531Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[1]R flush_numbers[0..12]: [138, 137, 136, 138, 138, 138, 138, 138, 138, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819296135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[1]R generation[0..12]: [69, 68, 68, 69, 69, 69, 69, 69, 69, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819311228Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819327631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[2]R flush_numbers[0..12]: [138, 137, 136, 138, 138, 138, 138, 138, 138, 131, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819343284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[2]R generation[0..12]: [69, 68, 68, 69, 69, 69, 69, 69, 69, 65, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819358438Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819374911Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Max found gen is 70","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819391453Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Generation requested: 70 >= found:70","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819405277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Next flush: 139","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819418362Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819431756Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"":"downstairs","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.81944602Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819459944Z","Wait for a query_work_queue command to finish before sending IO hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"0","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819479975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"1","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819493689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"client":"2","":"downstairs","session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819508083Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819521267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"358c38a3-d3a6-4d2d-b5fa-974f23b216ed is now active with session: 15aaf551-b1db-45f0-94dc-94d92fd20707","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819535961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819548765Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480,"session_id":"15aaf551-b1db-45f0-94dc-94d92fd20707"} {"msg":"The guest has finished waiting for activation with:70","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:51:28.819562099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} {"msg":"Activated sub_volume 358c38a3-d3a6-4d2d-b5fa-974f23b216ed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:28.819574534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 21 len 32768 data: 5 5 7 8 6 6 6 7 02/30 Read block 293 len 24576 03/30 Read block 14 len 8192 04/30 Write block 304 len 16384 data: 5 5 4 4 05/30 Write block 410 len 20480 data: 8 9 8 8 7 06/30 Read block 278 len 28672 07/30 Read block 440 len 20480 08/30 Read block 391 len 32768 09/30 Flush 10/30 Write block 96 len 12288 data: 8 9 8 11/30 Write block 403 len 8192 data: 7 8 12/30 Read block 260 len 16384 13/30 Read block 484 len 20480 14/30 Read block 359 len 12288 15/30 Read block 460 len 40960 16/30 Read block 16 len 28672 17/30 Flush 18/30 Write block 243 len 20480 data: 5 4 4 3 2 19/30 Read block 65 len 32768 20/30 Read block 125 len 28672 21/30 Read block 267 len 8192 22/30 Read block 543 len 12288 23/30 Write block 182 len 40960 data: 7 7 6 7 8 7 6 6 7 10 24/30 Flush 25/30 Flush 26/30 Write block 588 len 40960 data: 7 6 6 6 7 6 5 4 3 3 27/30 Read block 13 len 8192 28/30 Read block 546 len 16384 29/30 Read block 146 len 32768 30/30 Write block 123 len 32768 data: 9 8 11 8 6 6 6 6 ---------------------------------------------------------------- Crucible gen:70 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Sent Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 5 Done Sent Done false 1005 Acked Read 7 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Read 8 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 2 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Read 5 Done Sent Done false 1013 Acked Read 3 Done Sent Done false 1014 Acked Read 10 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Read 8 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Read 2 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Write 10 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 10 Done Sent Done false 1026 Acked Read 2 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:28.832161169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1480} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 138 138 138 F F F 1 020-039 70 68 70 139 137 139 F F F <--- 2 040-059 68 68 68 136 136 136 F F F 3 060-079 69 69 69 138 138 138 F F F 4 080-099 70 69 70 140 138 140 F F F <--- 5 100-119 69 69 69 138 138 138 F F F 6 120-139 69 69 69 138 138 138 T F T <--- 7 140-159 69 69 69 138 138 138 F F F 8 160-179 69 69 69 138 138 138 F F F 9 180-199 70 65 70 141 131 141 F F F <--- 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 70 65 70 141 131 141 F F F <--- 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 70 69 70 139 138 139 F F F <--- 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 68 68 68 136 136 136 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 68 70 140 136 140 F F F <--- 21 420-439 64 64 64 130 130 130 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 69 69 69 138 138 138 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 66 66 66 132 132 132 F F F 28 560-579 69 69 69 138 138 138 F F F 29 580-599 69 69 69 138 138 138 T F T <--- Max gen: 70, Max flush: 141 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 35 Verifying data now No change after verify Loop: 35 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 69 69 69 138 138 138 F F F 1 020-039 70 70 70 139 139 139 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 69 69 69 138 138 138 F F F 4 080-099 70 70 70 140 140 140 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 71 71 71 142 142 142 F F F 7 140-159 69 69 69 138 138 138 F F F 8 160-179 69 69 69 138 138 138 F F F 9 180-199 70 70 70 141 141 141 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 70 70 70 141 141 141 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 70 70 70 139 139 139 F F F 16 320-339 65 65 65 131 131 131 F F F 17 340-359 63 63 63 129 129 129 F F F 18 360-379 68 68 68 136 136 136 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 70 70 140 140 140 F F F 21 420-439 64 64 64 130 130 130 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 69 69 69 138 138 138 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 66 66 66 132 132 132 F F F 28 560-579 69 69 69 138 138 138 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 71, Max flush: 142 Begin loop 36 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.142587683Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.154298017Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.154619018Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.154659131Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.154671366Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.154689428Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} {"msg":"Crucible 61e1b311-44a1-4356-9f21-30ec5d57053f has session id: c2fcb49f-f790-4634-975f-dedb61698d79","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155004181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Upstairs opts: Upstairs UUID: 61e1b311-44a1-4356-9f21-30ec5d57053f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155051021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Crucible stats registered with UUID: 61e1b311-44a1-4356-9f21-30ec5d57053f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155066794Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:33.155080148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155250475Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155277743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155292736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155415323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"task":"crutest"} {"msg":"The guest has requested activation with gen:72","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155436714Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} {"msg":"61e1b311-44a1-4356-9f21-30ec5d57053f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155480935Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155495288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"0","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155512521Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"1","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155534282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"2","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155548106Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155562359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155576723Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"ds_connection connected from Ok(127.0.0.1:34305)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155621174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"2","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"ds_connection connected from Ok(127.0.0.1:54691)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155665705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"0","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"ds_connection connected from Ok(127.0.0.1:52166)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.155802125Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"io task","client":"1","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156395698Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"2","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156424716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"0","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156441818Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"1","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.1567845Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156799393Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[0]R flush_numbers[0..12]: [138, 139, 136, 138, 140, 138, 142, 138, 138, 141, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156833898Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[0]R generation[0..12]: [69, 70, 68, 69, 70, 69, 71, 69, 69, 70, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156851251Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156870352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[1]R flush_numbers[0..12]: [138, 139, 136, 138, 140, 138, 142, 138, 138, 141, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156886615Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[1]R generation[0..12]: [69, 70, 68, 69, 70, 69, 71, 69, 69, 70, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156901979Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156918302Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[2]R flush_numbers[0..12]: [138, 139, 136, 138, 140, 138, 142, 138, 138, 141, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156934225Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[2]R generation[0..12]: [69, 70, 68, 69, 70, 69, 71, 69, 69, 70, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156949468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156965961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Max found gen is 72","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.156979695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Generation requested: 72 >= found:72","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:51:33.156993879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Next flush: 143","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157012681Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157026375Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"":"downstairs","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157040269Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157052783Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"0","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157066647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"1","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157082101Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"client":"2","":"downstairs","session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157098873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157112348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"61e1b311-44a1-4356-9f21-30ec5d57053f is now active with session: c2fcb49f-f790-4634-975f-dedb61698d79","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157126891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157139666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488,"session_id":"c2fcb49f-f790-4634-975f-dedb61698d79"} {"msg":"The guest has finished waiting for activation with:72","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.15715297Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} {"msg":"Activated sub_volume 61e1b311-44a1-4356-9f21-30ec5d57053f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:33.157164375Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 20480 data: 7 7 8 6 6 02/30 Read block 449 len 24576 03/30 Write block 342 len 24576 data: 9 9 10 9 8 6 04/30 Read block 310 len 8192 05/30 Read block 443 len 40960 06/30 Write block 339 len 4096 data: 7 07/30 Read block 430 len 4096 08/30 Write block 74 len 20480 data: 5 6 5 5 7 09/30 Read block 276 len 40960 10/30 Flush 11/30 Write block 63 len 4096 data: 6 12/30 Read block 122 len 32768 13/30 Read block 341 len 28672 14/30 Flush 15/30 Write block 7 len 32768 data: 5 6 6 6 5 3 3 5 16/30 Write block 249 len 20480 data: 3 4 5 5 6 17/30 Read block 69 len 28672 18/30 Read block 484 len 20480 19/30 Read block 337 len 36864 20/30 Read block 252 len 40960 21/30 Write block 556 len 40960 data: 5 4 5 5 5 5 4 4 4 6 22/30 Write block 20 len 16384 data: 4 6 6 8 23/30 Write block 567 len 16384 data: 5 6 7 6 24/30 Write block 357 len 28672 data: 10 9 8 6 8 5 6 25/30 Flush 26/30 Write block 548 len 16384 data: 7 7 7 7 27/30 Write block 248 len 28672 data: 2 4 5 6 6 7 6 28/30 Read block 355 len 36864 29/30 Write block 548 len 12288 data: 8 8 8 30/30 Write block 500 len 36864 data: 9 9 8 5 6 5 6 6 7 ---------------------------------------------------------------- Crucible gen:72 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 6 Done Sent Done false 1002 Acked Write 6 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Read 10 Done Sent Done false 1005 Acked Write 1 Done Sent Done false 1006 Acked Read 1 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 10 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Read 8 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 10 Done Sent Done false 1020 Acked Write 10 Done Sent Done false 1021 Acked Write 4 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Write 7 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 4 Done Sent Done false 1026 Acked Write 7 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 3 Done Sent Done false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 29 1 31 Done 29 1 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:33.16912802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1488} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 69 72 145 138 145 F F F <--- 1 020-039 72 70 72 145 139 145 F F F <--- 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 69 72 144 138 144 F F F <--- 4 080-099 70 70 70 140 140 140 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 71 71 71 142 142 142 F F F 7 140-159 69 69 69 138 138 138 F F F 8 160-179 69 69 69 138 138 138 F F F 9 180-199 70 70 70 141 141 141 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 72 70 72 145 141 145 T F T <--- 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 70 70 70 139 139 139 F F F 16 320-339 72 65 72 143 131 143 F T F <--- 17 340-359 72 63 72 145 129 145 F T F <--- 18 360-379 72 68 72 145 136 145 F F F <--- 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 70 70 140 140 140 F F F 21 420-439 64 64 64 130 130 130 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 69 69 69 138 138 138 T F T <--- 26 520-539 69 69 69 138 138 138 F F F 27 540-559 72 66 72 145 132 145 T F T <--- 28 560-579 72 69 72 145 138 145 F F F <--- 29 580-599 71 71 71 142 142 142 F F F Max gen: 72, Max flush: 145 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 36 Verifying data now No change after verify Loop: 36 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 145 145 145 F F F 1 020-039 72 72 72 145 145 145 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 70 70 70 140 140 140 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 71 71 71 142 142 142 F F F 7 140-159 69 69 69 138 138 138 F F F 8 160-179 69 69 69 138 138 138 F F F 9 180-199 70 70 70 141 141 141 F F F 10 200-219 65 65 65 131 131 131 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 73 73 73 146 146 146 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 F F F 15 300-319 70 70 70 139 139 139 F F F 16 320-339 72 72 72 143 143 143 F F F 17 340-359 72 72 72 145 145 145 F F F 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 70 70 140 140 140 F F F 21 420-439 64 64 64 130 130 130 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 62 62 62 128 128 128 F F F 25 500-519 73 73 73 146 146 146 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 73 73 73 146 146 146 F F F 28 560-579 72 72 72 145 145 145 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 73, Max flush: 146 Begin loop 37 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.512590396Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.524385904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.524700238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.524739571Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.524751736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.524769438Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} {"msg":"Crucible ac2a5c82-2195-4881-868a-4a851f9665b9 has session id: 82f30815-e5f8-4d22-a0cb-d467aa094604","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525132511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Upstairs opts: Upstairs UUID: ac2a5c82-2195-4881-868a-4a851f9665b9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525173353Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Crucible stats registered with UUID: ac2a5c82-2195-4881-868a-4a851f9665b9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525191875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:37.525205499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525371877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525406812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525422545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525566043Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"task":"crutest"} {"msg":"The guest has requested activation with gen:74","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525582995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} {"msg":"ac2a5c82-2195-4881-868a-4a851f9665b9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525610873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525623898Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"0","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525637892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"1","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525651726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"2","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.52566508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525679474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525693687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"ds_connection connected from Ok(127.0.0.1:41193)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525718537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"1","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"ds_connection connected from Ok(127.0.0.1:34501)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.52573453Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"2","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"ds_connection connected from Ok(127.0.0.1:37933)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.525822262Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"io task","client":"0","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.526428959Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"1","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.526461835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"0","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.526827646Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.526879524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.52718855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"2","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[0]R flush_numbers[0..12]: [145, 145, 136, 144, 140, 138, 142, 138, 138, 141, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527576821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[0]R generation[0..12]: [72, 72, 68, 72, 70, 69, 71, 69, 69, 70, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527594274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527610967Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[1]R flush_numbers[0..12]: [145, 145, 136, 144, 140, 138, 142, 138, 138, 141, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.52762693Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[1]R generation[0..12]: [72, 72, 68, 72, 70, 69, 71, 69, 69, 70, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527641863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527658106Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[2]R flush_numbers[0..12]: [145, 145, 136, 144, 140, 138, 142, 138, 138, 141, 131, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527673839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[2]R generation[0..12]: [72, 72, 68, 72, 70, 69, 71, 69, 69, 70, 65, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527688713Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527707405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Max found gen is 74","v":0,"name":Wait for a query_work_queue command to finish before sending IO "crucible","level":30,"time":"2026-03-12T22:51:37.527726426Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Generation requested: 74 >= found:74","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.52774097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Next flush: 147","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527754204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527767559Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"":"downstairs","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527782122Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527795087Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"0","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527809201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"1","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:51:37.527823324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"client":"2","":"downstairs","session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527839358Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527853042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"ac2a5c82-2195-4881-868a-4a851f9665b9 is now active with session: 82f30815-e5f8-4d22-a0cb-d467aa094604","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527868115Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.52788059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496,"session_id":"82f30815-e5f8-4d22-a0cb-d467aa094604"} {"msg":"The guest has finished waiting for activation with:74","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527894264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} {"msg":"Activated sub_volume ac2a5c82-2195-4881-868a-4a851f9665b9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:37.527906189Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 154 len 32768 data: 6 7 7 5 6 8 11 11 02/30 Write block 136 len 12288 data: 10 9 8 03/30 Read block 283 len 4096 04/30 Write block 300 len 4096 data: 7 05/30 Read block 527 len 20480 06/30 Write block 425 len 28672 data: 7 5 2 3 5 5 5 07/30 Write block 161 len 24576 data: 12 10 7 5 4 5 08/30 Read block 205 len 24576 09/30 Flush 10/30 Write block 131 len 16384 data: 5 6 7 8 11/30 Write block 541 len 28672 data: 3 4 3 3 4 4 7 12/30 Write block 181 len 24576 data: 6 8 8 7 8 9 13/30 Write block 496 len 32768 data: 12 12 10 11 10 10 9 6 14/30 Flush 15/30 Write block 246 len 20480 data: 4 3 3 5 6 16/30 Write block 311 len 4096 data: 6 17/30 Read block 17 len 16384 18/30 Write block 489 len 32768 data: 10 6 5 6 7 8 11 13 19/30 Read block 106 len 8192 20/30 Read block 279 len 16384 21/30 Read block 135 len 16384 22/30 Read block 103 len 28672 23/30 Flush 24/30 Write block 280 len 4096 data: 3 25/30 Read block 452 len 28672 26/30 Read block 475 len 12288 27/30 Read block 22 len 40960 28/30 Write block 197 len 36864 data: 12 12 12 10 8 8 9 10 10 29/30 Read block 415 len 36864 30/30 Read block 581 len 32768 ---------------------------------------------------------------- Crucible gen:74 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Sent false 1001 Acked Write 3 Done Done Sent false 1002 Acked Read 1 Done Done Sent false 1003 Acked Write 1 Done Done Sent false 1004 Acked Read 5 Done Done Sent false 1005 Acked Write 7 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 1 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 8 Done Done Sent false 1018 Acked Read 2 Done Done Sent false 1019 Acked Read 4 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Write 9 Done Done Sent false 1028 Acked Read 9 Done Done Sent false 1029 Acked Read 8 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 30 30 Done 30 30 0 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:37.539820165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1496} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 145 145 145 F F F 1 020-039 72 72 72 145 145 145 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 70 70 70 140 140 140 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 74 74 71 148 148 142 F F F <--- 7 140-159 74 74 69 147 147 138 F F F <--- 8 160-179 74 74 69 147 147 138 F F F <--- 9 180-199 74 74 70 148 148 141 T T F <--- 10 200-219 65 65 65 131 131 131 T T F <--- 11 220-239 63 63 63 129 129 129 F F F 12 240-259 74 74 73 149 149 146 F F F <--- 13 260-279 64 64 64 130 130 130 F F F 14 280-299 66 66 66 133 133 133 T T F <--- 15 300-319 74 74 70 149 149 139 F F F <--- 16 320-339 72 72 72 143 143 143 F F F 17 340-359 72 72 72 145 145 145 F F F 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 70 70 140 140 140 F F F 21 420-439 74 74 64 147 147 130 F F F <--- 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 74 74 62 149 149 128 F F F <--- 25 500-519 74 74 73 148 148 146 F F F <--- 26 520-539 69 69 69 138 138 138 F F F 27 540-559 74 74 73 148 148 146 F F F <--- 28 560-579 72 72 72 145 145 145 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 74, Max flush: 149 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 37 Verifying data now No change after verify Loop: 37 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 72 72 72 145 145 145 F F F 1 020-039 72 72 72 145 145 145 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 70 70 70 140 140 140 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 74 74 74 148 148 148 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 75 75 75 150 150 150 F F F 11 220-239 63 63 63 129 129 129 F F F 12 240-259 74 74 74 149 149 149 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 75 75 75 150 150 150 F F F 15 300-319 74 74 74 149 149 149 F F F 16 320-339 72 72 72 143 143 143 F F F 17 340-359 72 72 72 145 145 145 F F F 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 70 70 140 140 140 F F F 21 420-439 74 74 74 147 147 147 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 74 74 74 149 149 149 F F F 25 500-519 74 74 74 148 148 148 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 74 74 74 148 148 148 F F F 28 560-579 72 72 72 145 145 145 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 75, Max flush: 150 Begin loop 38 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.859697342Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:41.870435108Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.870473391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.880851214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881147835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881188747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881200972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881218844Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} {"msg":"Crucible fde64c65-250a-483f-8190-b6389cd0a38a has session id: f276db21-f0ee-4b71-a776-acb794573309","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881567303Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Upstairs opts: Upstairs UUID: fde64c65-250a-483f-8190-b6389cd0a38a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.88162167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Crucible stats registered with UUID: fde64c65-250a-483f-8190-b6389cd0a38a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881637783Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:41.881651117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881820334Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881846662Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881861586Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881976796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"task":"crutest"} {"msg":"The guest has requested activation with gen:76","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.881994748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} {"msg":"fde64c65-250a-483f-8190-b6389cd0a38a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882025135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882038939Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"0","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882053033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"1","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882067147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"2","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882080911Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882095204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882109378Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"ds_connection connected from Ok(127.0.0.1:60387)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882124472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"1","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"ds_connection connected from Ok(127.0.0.1:58725)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882139315Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"2","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"ds_connection connected from Ok(127.0.0.1:40304)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882214023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"io task","client":"0","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882847568Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"1","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882954692Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"2","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.882988927Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"0","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883209182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883267077Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[0]R flush_numbers[0..12]: [145, 145, 136, 144, 140, 138, 148, 147, 147, 150, 150, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883401078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[0]R generation[0..12]: [72, 72, 68, 72, 70, 69, 74, 74, 74, 75, 75, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883417731Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883434214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[1]R flush_numbers[0..12]: [145, 145, 136, 144, 140, 138, 148, 147, 147, 150, 150, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883450257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[1]R generation[0..12]: [72, 72, 68, 72, 70, 69, 74, 74, 74, 75, 75, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883465431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883481883Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[2]R flush_numbers[0..12]: [145, 145, 136, 144, 140, 138, 148, 147, 147, 150, 150, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883497387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[2]R generation[0..12]: [72, 72, 68, 72, 70, 69, 74, 74, 74, 75, 75, 63]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.8835124Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883528793Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Max found gen is 76","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:51:41.883542257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Generation requested: 76 >= found:76","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883562079Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Next flush: 151","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883575673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883589277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"":"downstairs","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.88360538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883618464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"0","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883632678Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883649171Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"client":"2","":"downstairs","session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883663935Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883683026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"fde64c65-250a-483f-8190-b6389cd0a38a is now active with session: f276db21-f0ee-4b71-a776-acb794573309","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.88369811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883710714Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504,"session_id":"f276db21-f0ee-4b71-a776-acb794573309"} {"msg":"The guest has finished waiting for activation with:76","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883724078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} {"msg":"Activated sub_volume fde64c65-250a-483f-8190-b6389cd0a38a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:41.883735623Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 559 len 24576 data: 6 6 6 5 5 5 02/30 Write block 8 len 4096 data: 7 03/30 Flush 04/30 Write block 29 len 20480 data: 8 7 7 7 8 05/30 Read block 221 len 20480 06/30 Read block 476 len 8192 07/30 Flush 08/30 Write block 404 len 40960 data: 9 9 10 8 8 10 9 10 9 9 09/30 Write block 332 len 28672 data: 5 4 4 6 8 8 9 10/30 Read block 64 len 8192 11/30 Read block 189 len 40960 12/30 Read block 94 len 28672 13/30 Read block 11 len 28672 14/30 Write block 209 len 20480 data: 7 5 6 6 5 15/30 Read block 422 len 8192 16/30 Write block 571 len 16384 data: 4 5 3 4 17/30 Write block 237 len 36864 data: 5 6 7 7 7 5 6 5 5 18/30 Read block 386 len 32768 19/30 Read block 143 len 4096 20/30 Read block 300 len 8192 21/30 Read block 568 len 32768 22/30 Read block 120 len 40960 23/30 Write block 317 len 20480 data: 6 5 7 6 5 24/30 Read block 154 len 16384 25/30 Flush 26/30 Write block 88 len 4096 data: 5 27/30 Read block 408 len 20480 28/30 Write block 509 len 16384 data: 7 8 10 11 29/30 Read block 569 len 12288 30/30 Write block 37 len 8192 data: 6 6 ---------------------------------------------------------------- Crucible gen:76 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 5 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Read 2 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 10 Sent Done Done false 1008 Acked Write 7 Sent Done Done false 1009 Acked Read 2 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Read 7 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 5 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Read 2 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Read 5 Sent Done Done false 1027 Acked Write 4 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 0 26 26 52 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1002)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:41.896120276Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1504} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 151 151 151 F F F 1 020-039 72 76 76 145 152 152 F T T <--- 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 70 70 70 140 140 140 F T T <--- 5 100-119 69 69 69 138 138 138 F F F 6 120-139 74 74 74 148 148 148 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 75 76 76 150 153 153 F F F <--- 11 220-239 63 76 76 129 153 153 F F F <--- 12 240-259 74 76 76 149 153 153 F F F <--- 13 260-279 64 64 64 130 130 130 F F F 14 280-299 75 75 75 150 150 150 F F F 15 300-319 74 76 76 149 153 153 F F F <--- 16 320-339 72 76 76 143 153 153 F F F <--- 17 340-359 72 72 72 145 145 145 F F F 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 70 76 76 140 153 153 F F F <--- 21 420-439 74 74 74 147 147 147 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 74 74 74 149 149 149 F F F 25 500-519 74 74 74 148 148 148 F T T <--- 26 520-539 69 69 69 138 138 138 F F F 27 540-559 76 76 76 151 151 151 F F F 28 560-579 76 76 76 151 153 153 F F F <--- 29 580-599 71 71 71 142 142 142 F F F Max gen: 76, Max flush: 153 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 38 Verifying data now No change after verify Loop: 38 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 76 76 76 151 151 151 F F F 1 020-039 77 77 77 154 154 154 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 77 77 77 154 154 154 F F F 5 100-119 69 69 69 138 138 138 F F F 6 120-139 74 74 74 148 148 148 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 76 76 76 153 153 153 F F F 11 220-239 76 76 76 153 153 153 F F F 12 240-259 76 76 76 153 153 153 F F F 13 260-279 64 64 64 130 130 130 F F F 14 280-299 75 75 75 150 150 150 F F F 15 300-319 76 76 76 153 153 153 F F F 16 320-339 76 76 76 153 153 153 F F F 17 340-359 72 72 72 145 145 145 F F F 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 76 76 76 153 153 153 F F F 21 420-439 74 74 74 147 147 147 F F F 22 440-459 69 69 69 138 138 138 F F F 23 460-479 69 69 69 138 138 138 F F F 24 480-499 74 74 74 149 149 149 F F F 25 500-519 77 77 77 154 154 154 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 76 76 76 151 151 151 F F F 28 560-579 76 76 76 153 153 153 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 77, Max flush: 154 Begin loop 39 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.218994455Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231019214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231348951Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231388494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231400419Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231418151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} {"msg":"Crucible 414f8a0d-524b-4409-a27f-60fbd5016bc0 has session id: 78343e99-8620-49b1-bf4e-3f6e10d718fe","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231748638Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Upstairs opts: Upstairs UUID: 414f8a0d-524b-4409-a27f-60fbd5016bc0, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231828963Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Crucible stats registered with UUID: 414f8a0d-524b-4409-a27f-60fbd5016bc0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.231846326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:46.23185973Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232058034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.23209052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232105893Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232222393Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"task":"crutest"} {"msg":"The guest has requested activation with gen:78","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232239315Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} {"msg":"414f8a0d-524b-4409-a27f-60fbd5016bc0 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232269762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232283806Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"0","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.23229808Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"1","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232312004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"2","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232325698Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232339942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232354275Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:34645)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232371378Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"2","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:54830)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232487828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"1","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"ds_connection connected from Ok(127.0.0.1:59875)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.232515076Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"io task","client":"0","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233049034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"2","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233114546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"0","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233234104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"1","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233403161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233442434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[0]R flush_numbers[0..12]: [151, 154, 136, 144, 154, 138, 148, 147, 147, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233605913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[0]R generation[0..12]: [76, 77, 68, 72, 77, 69, 74, 74, 74, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233623315Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233639468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[1]R flush_numbers[0..12]: [151, 154, 136, 144, 154, 138, 148, 147, 147, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233654882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[1]R generation[0..12]: [76, 77, 68, 72, 77, 69, 74, 74, 74, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233670055Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233686228Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[2]R flush_numbers[0..12]: [151, 154, 136, 144, 154, 138, 148, 147, 147, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.23370496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[2]R generation[0..12]: [76, 77, 68, 72, 77, 69, 74, 74, 74, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233720253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233736536Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Max found gen is 78","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:51:46.233749971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Generation requested: 78 >= found:78","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233777758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Next flush: 155","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233790973Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233804427Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"":"downstairs","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233818201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233830566Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"0","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.23384458Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"1","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233861522Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"client":"2","":"downstairs","session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233875816Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.23388946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"414f8a0d-524b-4409-a27f-60fbd5016bc0 is now active with session: 78343e99-8620-49b1-bf4e-3f6e10d718fe","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233904574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233917228Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512,"session_id":"78343e99-8620-49b1-bf4e-3f6e10d718fe"} {"msg":"The guest has finished waiting for activation with:78","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233930642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} {"msg":"Activated sub_volume 414f8a0d-524b-4409-a27f-60fbd5016bc0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:46.233943717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 240 len 36864 data: 8 8 6 7 6 6 5 4 4 02/30 Write block 272 len 16384 data: 6 7 5 4 03/30 Read block 140 len 24576 04/30 Read block 323 len 4096 05/30 Read block 308 len 24576 06/30 Write block 502 len 24576 data: 10 7 7 6 7 7 07/30 Flush 08/30 Write block 478 len 32768 data: 4 6 8 9 11 10 9 8 09/30 Write block 435 len 40960 data: 7 7 8 9 8 8 8 6 6 6 10/30 Read block 372 len 4096 11/30 Read block 482 len 28672 12/30 Write block 10 len 40960 data: 7 6 4 4 6 6 7 6 5 3 13/30 Read block 170 len 40960 14/30 Read block 422 len 28672 15/30 Read block 460 len 20480 16/30 Read block 314 len 16384 17/30 Flush 18/30 Flush 19/30 Write block 550 len 4096 data: 9 20/30 Write block 464 len 12288 data: 7 6 6 21/30 Read block 423 len 20480 22/30 Write block 102 len 4096 data: 7 23/30 Read block 475 len 32768 24/30 Write block 353 len 16384 data: 8 8 8 9 25/30 Flush 26/30 Write block 477 len 24576 data: 3 5 7 9 10 12 27/30 Read block 373 len 40960 28/30 Write block 260 len 20480 data: 10 10 9 6 6 29/30 Write block 259 len 36864 data: 9 11 11 10 7 7 7 8 8 30/30 Read block 326 len 32768 ---------------------------------------------------------------- Crucible gen:78 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 4 Done Sent Done false 1002 Acked Read 6 Done Sent Done false 1003 Acked Read 1 Done Sent Done false 1004 Acked Read 6 Done Sent Done false 1005 Acked Write 6 Done Sent Done false 1006 Acked Flush 0 Done Sent Done false 1007 Acked Write 8 Done Sent Done false 1008 Acked Write 10 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Read 7 Done Sent Done false 1011 Acked Write 10 Done Sent Done false 1012 Acked Read 10 Done Sent Done false 1013 Acked Read 7 Done Sent Done false 1014 Acked Read 5 Done Sent Done false 1015 Acked Read 4 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 3 Done Sent Done false 1020 Acked Read 5 Done Sent Done false 1021 Acked Write 1 Done Sent Done false 1022 Acked Read 8 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 6 Done Sent Done false 1026 Acked Read 10 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Write 9 Done Sent Done false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:46.246748797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1512} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 76 78 156 151 156 F F F <--- 1 020-039 77 77 77 154 154 154 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 77 77 77 154 154 154 F F F 5 100-119 78 69 78 158 138 158 F F F <--- 6 120-139 74 74 74 148 148 148 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 76 76 76 153 153 153 F F F 11 220-239 76 76 76 153 153 153 F F F 12 240-259 78 76 78 155 153 155 T T T <--- 13 260-279 78 64 78 155 130 155 T F T <--- 14 280-299 75 75 75 150 150 150 F F F 15 300-319 76 76 76 153 153 153 F F F 16 320-339 76 76 76 153 153 153 F F F 17 340-359 78 72 78 158 145 158 F F F <--- 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 76 76 76 153 153 153 F F F 21 420-439 78 74 78 156 147 156 F F F <--- 22 440-459 78 69 78 156 138 156 F F F <--- 23 460-479 78 69 78 158 138 158 T F T <--- 24 480-499 78 74 78 156 149 156 T F T <--- 25 500-519 78 77 78 155 154 155 F F F <--- 26 520-539 69 69 69 138 138 138 F F F 27 540-559 78 76 78 158 151 158 F F F <--- 28 560-579 76 76 76 153 153 153 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 78, Max flush: 158 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 39 Verifying data now No change after verify Loop: 39 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 156 156 156 F F F 1 020-039 77 77 77 154 154 154 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 77 77 77 154 154 154 F F F 5 100-119 78 78 78 158 158 158 F F F 6 120-139 74 74 74 148 148 148 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 76 76 76 153 153 153 F F F 11 220-239 76 76 76 153 153 153 F F F 12 240-259 79 79 79 159 159 159 F F F 13 260-279 79 79 79 159 159 159 F F F 14 280-299 75 75 75 150 150 150 F F F 15 300-319 76 76 76 153 153 153 F F F 16 320-339 76 76 76 153 153 153 F F F 17 340-359 78 78 78 158 158 158 F F F 18 360-379 72 72 72 145 145 145 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 76 76 76 153 153 153 F F F 21 420-439 78 78 78 156 156 156 F F F 22 440-459 78 78 78 156 156 156 F F F 23 460-479 79 79 79 159 159 159 F F F 24 480-499 79 79 79 159 159 159 F F F 25 500-519 78 78 78 155 155 155 F F F 26 520-539 69 69 69 138 138 138 F F F 27 540-559 78 78 78 158 158 158 F F F 28 560-579 76 76 76 153 153 153 F F F 29 580-599 71 71 71 142 142 142 F F F Max gen: 79, Max flush: 159 Begin loop 40 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.564205794Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.575969656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576304021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576342554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576354799Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576372811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} {"msg":"Crucible a78c45fb-cf93-4f7f-832b-a7a4d0a700eb has session id: c3ff5e94-ca55-4757-8e2b-340b70f65734","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576715912Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Upstairs opts: Upstairs UUID: a78c45fb-cf93-4f7f-832b-a7a4d0a700eb, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576750617Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Crucible stats registered with UUID: a78c45fb-cf93-4f7f-832b-a7a4d0a700eb","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57676643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:50.576780114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576955408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.576991183Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577007046Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57713719Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"task":"crutest"} {"msg":"The guest has requested activation with gen:80","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577153083Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} {"msg":"a78c45fb-cf93-4f7f-832b-a7a4d0a700eb active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577169855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57718279Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"0","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577196694Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"1","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577210518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"2","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577224142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577238036Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.5772522Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"ds_connection connected from Ok(127.0.0.1:44543)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57732135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"0","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"ds_connection connected from Ok(127.0.0.1:36998)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577441967Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"2","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"ds_connection connected from Ok(127.0.0.1:43697)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.577461149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"io task","client":"1","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578095184Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"0","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578146162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"1","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578170592Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"2","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578445633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578483956Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[0]R flush_numbers[0..12]: [156, 154, 136, 144, 154, 158, 148, 147, 147, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578569509Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[0]R generation[0..12]: [78, 77, 68, 72, 77, 78, 74, 74, 74, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578586981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578603674Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[1]R flush_numbers[0..12]: [156, 154, 136, 144, 154, 158, 148, 147, 147, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578619687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[1]R generation[0..12]: [78, 77, 68, 72, 77, 78, 74, 74, 74, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57863485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578651533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[2]R flush_numbers[0..12]: [156, 154, 136, 144, 154, 158, 148, 147, 147, 150, 153, 153]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578667326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[2]R generation[0..12]: [78, 77, 68, 72, 77, 78, 74, 74, 74, 75, 76, 76]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57868248Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578698773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Max found gen is 80","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578712177Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Generation requested: 80 >= found:80","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:51:50.578726231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Next flush: 160","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578745063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578758687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"":"downstairs","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57877456Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578787434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"0","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578801798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"1","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578815952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"client":"2","":"downstairs","session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"All required reconciliation work is completed"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578833994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578847888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"a78c45fb-cf93-4f7f-832b-a7a4d0a700eb is now active with session: c3ff5e94-ca55-4757-8e2b-340b70f65734","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578862932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578875716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520,"session_id":"c3ff5e94-ca55-4757-8e2b-340b70f65734"} {"msg":"The guest has finished waiting for activation with:80","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.57888891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} {"msg":"Activated sub_volume a78c45fb-cf93-4f7f-832b-a7a4d0a700eb","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:50.578900375Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 449 len 24576 data: 4 5 6 8 8 8 03/30 Read block 234 len 12288 04/30 Write block 92 len 8192 data: 8 8 05/30 Read block 228 len 40960 06/30 Flush 07/30 Flush 08/30 Write block 262 len 20480 data: 11 8 8 8 9 09/30 Read block 11 len 12288 10/30 Write block 134 len 4096 data: 9 11/30 Read block 342 len 12288 12/30 Write block 451 len 24576 data: 7 9 9 9 8 9 13/30 Write block 499 len 36864 data: 12 11 11 11 8 8 7 8 8 14/30 Write block 525 len 20480 data: 6 4 5 5 7 15/30 Read block 555 len 20480 16/30 Write block 309 len 16384 data: 7 6 7 7 17/30 Read block 318 len 36864 18/30 Write block 228 len 40960 data: 4 5 5 4 6 7 6 6 6 6 19/30 Flush 20/30 Flush 21/30 Flush 22/30 Flush 23/30 Write block 3 len 40960 data: 2 2 4 5 6 8 7 8 7 5 24/30 Write block 356 len 24576 data: 10 11 10 9 7 9 25/30 Write block 436 len 16384 data: 8 9 10 9 26/30 Write block 518 len 40960 data: 9 8 8 6 5 5 5 7 5 6 27/30 Write block 578 len 12288 data: 7 8 8 28/30 Write block 519 len 40960 data: 9 9 7 6 6 6 8 6 7 6 29/30 Write block 200 len 32768 data: 11 9 9 10 11 11 8 7 30/30 Read block 265 len 12288 ---------------------------------------------------------------- Crucible gen:80 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Write 9 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Read 9 Done Done Sent false 1017 Acked Write 10 Done Done Sent false 1018 Acked Flush 0 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Flush 0 Done Done Sent false 1021 Acked Flush 0 Done Done Sent false 1022 Acked Write 10 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Write 4 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 8 Done Sent Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 23 25 Done 23 21 0 44 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) Some(JobId(1021)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:50.591087733Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1520} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 78 78 78 156 156 156 T T F <--- 1 020-039 77 77 77 154 154 154 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 80 80 80 161 161 161 F F F 5 100-119 78 78 78 158 158 158 F F F 6 120-139 80 80 74 163 163 148 F F F <--- 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 76 76 76 153 153 153 T T F <--- 11 220-239 80 80 76 163 163 153 F F F <--- 12 240-259 79 79 79 159 159 159 F F F 13 260-279 80 80 79 163 163 159 F F F <--- 14 280-299 75 75 75 150 150 150 F F F 15 300-319 80 80 76 163 163 153 F F F <--- 16 320-339 76 76 76 153 153 153 F F F 17 340-359 78 78 78 158 158 158 T T F <--- 18 360-379 72 72 72 145 145 145 T T F <--- 19 380-399 68 68 68 137 137 137 F F F 20 400-419 76 76 76 153 153 153 F F F 21 420-439 78 78 78 156 156 156 T T F <--- 22 440-459 80 80 80 163 163 161 F F F <--- 23 460-479 79 79 79 159 159 159 F F F 24 480-499 80 80 79 163 163 159 F F F <--- 25 500-519 80 80 78 163 163 155 T T F <--- 26 520-539 80 80 69 163 163 138 T T F <--- 27 540-559 78 78 78 158 158 158 F F F 28 560-579 76 76 76 153 153 153 T T F <--- 29 580-599 71 71 71 142 142 142 T T F <--- Max gen: 80, Max flush: 163 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 40 Verifying data now No change after verify Loop: 40 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 164 164 164 F F F 1 020-039 77 77 77 154 154 154 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 F F F 4 080-099 80 80 80 161 161 161 F F F 5 100-119 78 78 78 158 158 158 F F F 6 120-139 80 80 80 163 163 163 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 81 81 81 164 164 164 F F F 11 220-239 80 80 80 163 163 163 F F F 12 240-259 79 79 79 159 159 159 F F F 13 260-279 80 80 80 163 163 163 F F F 14 280-299 75 75 75 150 150 150 F F F 15 300-319 80 80 80 163 163 163 F F F 16 320-339 76 76 76 153 153 153 F F F 17 340-359 81 81 81 164 164 164 F F F 18 360-379 81 81 81 164 164 164 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 76 76 76 153 153 153 F F F 21 420-439 81 81 81 164 164 164 F F F 22 440-459 80 80 80 163 163 163 F F F 23 460-479 79 79 79 159 159 159 F F F 24 480-499 80 80 80 163 163 163 F F F 25 500-519 81 81 81 164 164 164 F F F 26 520-539 81 81 81 164 164 164 F F F 27 540-559 78 78 78 158 158 158 F F F 28 560-579 81 81 81 164 164 164 F F F 29 580-599 81 81 81 164 164 164 F F F Max gen: 81, Max flush: 164 Begin loop 41 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.91966704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.931357243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.931683162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.931719996Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.931732101Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.931749823Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} {"msg":"Crucible 06402e8d-8b9b-4254-9db2-520c98b3afde has session id: 8dc96bd7-c436-4a8c-b4f8-bbf579c08c89","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932107798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Upstairs opts: Upstairs UUID: 06402e8d-8b9b-4254-9db2-520c98b3afde, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932165003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Crucible stats registered with UUID: 06402e8d-8b9b-4254-9db2-520c98b3afde","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932181276Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:54.9321945Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932354181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93237906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932393584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932528985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"task":"crutest"} {"msg":"The guest has requested activation with gen:82","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932546628Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} {"msg":"06402e8d-8b9b-4254-9db2-520c98b3afde active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932561231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932573676Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"0","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93258762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"1","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932601444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"2","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932614948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932629082Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932643546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:64838)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932707118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"0","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:47192)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93284139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"1","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"ds_connection connected from Ok(127.0.0.1:36771)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.932869618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"io task","client":"2","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933438961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"1","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933488569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"2","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933507051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"0","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93378704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933827313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[0]R flush_numbers[0..12]: [164, 154, 136, 144, 161, 158, 163, 147, 147, 150, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933889695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[0]R generation[0..12]: [81, 77, 68, 72, 80, 78, 80, 74, 74, 75, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933906308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933922941Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[1]R flush_numbers[0..12]: [164, 154, 136, 144, 161, 158, 163, 147, 147, 150, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933945241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[1]R generation[0..12]: [81, 77, 68, 72, 80, 78, 80, 74, 74, 75, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933960675Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.933976838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[2]R flush_numbers[0..12]: [164, 154, 136, 144, 161, 158, 163, 147, 147, 150, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93399509Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[2]R generation[0..12]: [81, 77, 68, 72, 80, 78, 80, 74, 74, 75, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934010533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934026786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Max found gen is 82","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93404051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 82 >= found:82","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934060002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Next flush: 165","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934073536Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.93408661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"":"downstairs","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934100614Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934113389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"0","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934127572Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"1","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934141906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"client":"2","":"downstairs","session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"All required reconciliation work is completed"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934159129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934172883Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"06402e8d-8b9b-4254-9db2-520c98b3afde is now active with session: 8dc96bd7-c436-4a8c-b4f8-bbf579c08c89","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934187946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934200621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528,"session_id":"8dc96bd7-c436-4a8c-b4f8-bbf579c08c89"} {"msg":"The guest has finished waiting for activation with:82","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934213805Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} {"msg":"Activated sub_volume 06402e8d-8b9b-4254-9db2-520c98b3afde","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:54.934227199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 404 len 8192 data: 10 10 02/30 Write block 83 len 20480 data: 7 8 9 8 6 03/30 Read block 124 len 16384 04/30 Read block 152 len 12288 05/30 Write block 478 len 36864 data: 6 8 10 11 13 11 10 9 8 06/30 Read block 471 len 36864 07/30 Read block 486 len 12288 08/30 Write block 439 len 20480 data: 10 9 9 7 7 09/30 Write block 501 len 16384 data: 12 12 9 9 10/30 Write block 280 len 16384 data: 4 2 5 4 11/30 Write block 554 len 40960 data: 6 4 6 5 6 7 7 7 6 6 12/30 Read block 542 len 16384 13/30 Read block 224 len 16384 14/30 Flush 15/30 Write block 103 len 16384 data: 6 7 5 5 16/30 Read block 333 len 12288 17/30 Read block 149 len 24576 18/30 Read block 526 len 28672 19/30 Write block 79 len 40960 data: 7 8 6 7 8 9 10 9 7 6 20/30 Read block 277 len 40960 21/30 Write block 25 len 40960 data: 7 7 7 8 9 8 8 8 9 8 22/30 Write block 98 len 28672 data: 9 8 10 9 8 7 8 23/30 Read block 88 len 12288 24/30 Write block 501 len 32768 data: 13 13 10 10 8 9 9 8 25/30 Write block 506 len 28672 data: 10 10 9 8 9 11 12 26/30 Write block 503 len 4096 data: 11 27/30 Write block 271 len 8192 data: 5 7 28/30 Read block 582 len 24576 29/30 Write block 122 len 28672 data: 9 10 9 12 9 7 7 30/30 Read block 31 len 24576 ---------------------------------------------------------------- Crucible gen:82 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 5 Done Done Sent false 1002 Acked Read 4 Done Done Sent false 1003 Acked Read 3 Done Done Sent false 1004 Acked Write 9 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Write 4 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 3 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Read 7 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Read 10 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 7 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 2 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1013)) Some(JobId(1013)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:54.94680027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1528} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 164 164 164 F F F 1 020-039 77 77 77 154 154 154 T T F <--- 2 040-059 68 68 68 136 136 136 F F F 3 060-079 72 72 72 144 144 144 T T F <--- 4 080-099 82 82 80 165 165 161 T T F <--- 5 100-119 78 78 78 158 158 158 T T F <--- 6 120-139 80 80 80 163 163 163 T T F <--- 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 81 81 81 164 164 164 F F F 11 220-239 80 80 80 163 163 163 F F F 12 240-259 79 79 79 159 159 159 F F F 13 260-279 80 80 80 163 163 163 T T F <--- 14 280-299 82 82 75 165 165 150 F F F <--- 15 300-319 80 80 80 163 163 163 F F F 16 320-339 76 76 76 153 153 153 F F F 17 340-359 81 81 81 164 164 164 F F F 18 360-379 81 81 81 164 164 164 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 82 82 76 165 165 153 F F T <--- 21 420-439 82 82 81 165 165 164 F F F <--- 22 440-459 82 82 80 165 165 163 F F F <--- 23 460-479 82 82 79 165 165 159 F F F <--- 24 480-499 82 82 80 165 165 163 F F F <--- 25 500-519 82 82 81 165 165 164 T T F <--- 26 520-539 81 81 81 164 164 164 F F F 27 540-559 82 82 78 165 165 158 F F F <--- 28 560-579 82 82 81 165 165 164 F F F <--- 29 580-599 81 81 81 164 164 164 F F F Max gen: 82, Max flush: 165 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 41 Verifying data now No change after verify Loop: 41 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 164 164 164 F F F 1 020-039 83 83 83 166 166 166 F F F 2 040-059 68 68 68 136 136 136 F F F 3 060-079 83 83 83 166 166 166 F F F 4 080-099 83 83 83 166 166 166 F F F 5 100-119 83 83 83 166 166 166 F F F 6 120-139 83 83 83 166 166 166 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 74 74 74 147 147 147 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 81 81 81 164 164 164 F F F 11 220-239 80 80 80 163 163 163 F F F 12 240-259 79 79 79 159 159 159 F F F 13 260-279 83 83 83 166 166 166 F F F 14 280-299 82 82 82 165 165 165 F F F 15 300-319 80 80 80 163 163 163 F F F 16 320-339 76 76 76 153 153 153 F F F 17 340-359 81 81 81 164 164 164 F F F 18 360-379 81 81 81 164 164 164 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 82 82 82 165 165 165 F F F 21 420-439 82 82 82 165 165 165 F F F 22 440-459 82 82 82 165 165 165 F F F 23 460-479 82 82 82 165 165 165 F F F 24 480-499 82 82 82 165 165 165 F F F 25 500-519 83 83 83 166 166 166 F F F 26 520-539 81 81 81 164 164 164 F F F 27 540-559 82 82 82 165 165 165 F F F 28 560-579 82 82 82 165 165 165 F F F 29 580-599 81 81 81 164 164 164 F F F Max gen: 83, Max flush: 166 Begin loop 42 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.276261194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.287951028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.288277047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.288318209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.288330404Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.288348306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} {"msg":"Crucible d65d4287-4136-4a82-a73c-ee7ee0f7cc73 has session id: 27139852-9129-4724-835f-b8338d82cc77","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.288740626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Upstairs opts: Upstairs UUID: d65d4287-4136-4a82-a73c-ee7ee0f7cc73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.28877752Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Crucible stats registered with UUID: d65d4287-4136-4a82-a73c-ee7ee0f7cc73","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.288793373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:59.288807097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289003692Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289040056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289055879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289203425Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"task":"crutest"} {"msg":"The guest has requested activation with gen:84","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289220938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} {"msg":"d65d4287-4136-4a82-a73c-ee7ee0f7cc73 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289236321Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289249385Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"0","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289263339Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"1","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289277633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"2","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289291317Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289305891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289320345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"ds_connection connected from Ok(127.0.0.1:49385)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289388695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"2","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"ds_connection connected from Ok(127.0.0.1:50199)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289532343Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"1","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"ds_connection connected from Ok(127.0.0.1:54882)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.289555053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"io task","client":"0","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.29013899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"2","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290166718Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"0","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290183001Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"1","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.2905301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.29055492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[0]R flush_numbers[0..12]: [164, 166, 136, 166, 166, 166, 166, 147, 147, 150, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290570293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[0]R generation[0..12]: [81, 83, 68, 83, 83, 83, 83, 74, 74, 75, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290586616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290603468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[1]R flush_numbers[0..12]: [164, 166, 136, 166, 166, 166, 166, 147, 147, 150, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290619452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[1]R generation[0..12]: [81, 83, 68, 83, 83, 83, 83, 74, 74, 75, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290634995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290651258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[2]R flush_numbers[0..12]: [164, 166, 136, 166, 166, 166, 166, 147, 147, 150, 164, 163]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290667321Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[2]R generation[0..12]: [81, 83, 68, 83, 83, 83, 83, 74, 74, 75, 81, 80]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290682304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290698867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Max found gen is 84","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290712361Z"Wait for a query_work_queue command to finish before sending IO ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Generation requested: 84 >= found:84","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290731993Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Next flush: 167","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290745127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290758491Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"":"downstairs","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290775874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290788778Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"0","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290803132Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"1","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:51:59.290817246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"client":"2","":"downstairs","session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290833489Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290847133Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"d65d4287-4136-4a82-a73c-ee7ee0f7cc73 is now active with session: 27139852-9129-4724-835f-b8338d82cc77","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290862006Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290874801Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536,"session_id":"27139852-9129-4724-835f-b8338d82cc77"} {"msg":"The guest has finished waiting for activation with:84","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.290888155Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} {"msg":"Activated sub_volume d65d4287-4136-4a82-a73c-ee7ee0f7cc73","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:51:59.2908999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 589 len 4096 data: 7 02/30 Flush 03/30 Write block 572 len 16384 data: 6 4 5 4 04/30 Write block 318 len 24576 data: 6 8 7 6 3 4 05/30 Write block 284 len 16384 data: 8 11 14 15 06/30 Write block 237 len 32768 data: 7 7 8 9 9 7 8 7 07/30 Write block 346 len 36864 data: 9 7 5 9 10 9 10 9 9 08/30 Write block 41 len 20480 data: 7 6 5 6 6 09/30 Read block 204 len 36864 10/30 Write block 212 len 20480 data: 7 6 5 4 3 11/30 Read block 571 len 16384 12/30 Write block 291 len 28672 data: 13 9 9 8 8 8 9 13/30 Write block 168 len 40960 data: 9 10 11 12 10 7 7 7 6 5 14/30 Read block 479 len 40960 15/30 Read block 15 len 24576 16/30 Write block 319 len 28672 data: 9 8 7 4 5 4 4 17/30 Write block 270 len 8192 data: 5 6 18/30 Read block 535 len 20480 19/30 Read block 84 len 36864 20/30 Read block 595 len 20480 21/30 Read block 573 len 12288 22/30 Flush 23/30 Write block 73 len 16384 data: 3 6 7 6 24/30 Read block 164 len 12288 25/30 Write block 518 len 20480 data: 10 10 10 8 7 26/30 Flush 27/30 Write block 4 len 24576 data: 3 5 6 7 9 8 28/30 Read block 39 len 20480 29/30 Read block 316 len 32768 30/30 Write block 542 len 16384 data: 5 4 4 5 ---------------------------------------------------------------- Crucible gen:84 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Sent Done false 1001 Acked Flush 0 Done Sent Done false 1002 Acked Write 4 Done Sent Done false 1003 Acked Write 6 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Write 8 Done Sent Done false 1006 Acked Write 9 Done Sent Done false 1007 Acked Write 5 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 5 Done Sent Done false 1010 Acked Read 4 Done Sent Done false 1011 Acked Write 7 Done Sent Done false 1012 Acked Write 10 Done Sent Done false 1013 Acked Read 10 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Write 7 Done Sent Done false 1016 Acked Write 2 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 9 Done Sent Done false 1019 Acked Read 5 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Read 3 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Flush 0 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 5 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) None Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:51:59.303331562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1536} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 81 81 81 164 164 164 T F T <--- 1 020-039 83 83 83 166 166 166 F F F 2 040-059 84 68 84 168 136 168 F F F <--- 3 060-079 84 83 84 169 166 169 F F F <--- 4 080-099 83 83 83 166 166 166 F F F 5 100-119 83 83 83 166 166 166 F F F 6 120-139 83 83 83 166 166 166 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 84 74 84 168 147 168 F F F <--- 9 180-199 75 75 75 150 150 150 F F F 10 200-219 84 81 84 168 164 168 F F F <--- 11 220-239 84 80 84 168 163 168 F F F <--- 12 240-259 84 79 84 168 159 168 F F F <--- 13 260-279 84 83 84 168 166 168 F F F <--- 14 280-299 84 82 84 168 165 168 F F F <--- 15 300-319 84 80 84 168 163 168 F F F <--- 16 320-339 84 76 84 168 153 168 F F F <--- 17 340-359 84 81 84 168 164 168 F F F <--- 18 360-379 81 81 81 164 164 164 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 82 82 82 165 165 165 F F F 21 420-439 82 82 82 165 165 165 F F F 22 440-459 82 82 82 165 165 165 F F F 23 460-479 82 82 82 165 165 165 F F F 24 480-499 82 82 82 165 165 165 F F F 25 500-519 84 83 84 169 166 169 F F F <--- 26 520-539 84 81 84 169 164 169 F F F <--- 27 540-559 82 82 82 165 165 165 T F T <--- 28 560-579 84 82 84 168 165 168 F F F <--- 29 580-599 84 81 84 167 164 167 F F F <--- Max gen: 84, Max flush: 169 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 42 Verifying data now No change after verify Loop: 42 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 170 170 170 F F F 1 020-039 83 83 83 166 166 166 F F F 2 040-059 84 84 84 168 168 168 F F F 3 060-079 84 84 84 169 169 169 F F F 4 080-099 83 83 83 166 166 166 F F F 5 100-119 83 83 83 166 166 166 F F F 6 120-139 83 83 83 166 166 166 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 84 84 84 168 168 168 F F F 9 180-199 75 75 75 150 150 150 F F F 10 200-219 84 84 84 168 168 168 F F F 11 220-239 84 84 84 168 168 168 F F F 12 240-259 84 84 84 168 168 168 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 84 84 84 168 168 168 F F F 16 320-339 84 84 84 168 168 168 F F F 17 340-359 84 84 84 168 168 168 F F F 18 360-379 81 81 81 164 164 164 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 82 82 82 165 165 165 F F F 21 420-439 82 82 82 165 165 165 F F F 22 440-459 82 82 82 165 165 165 F F F 23 460-479 82 82 82 165 165 165 F F F 24 480-499 82 82 82 165 165 165 F F F 25 500-519 84 84 84 169 169 169 F F F 26 520-539 84 84 84 169 169 169 F F F 27 540-559 85 85 85 170 170 170 F F F 28 560-579 84 84 84 168 168 168 F F F 29 580-599 84 84 84 167 167 167 F F F Max gen: 85, Max flush: 170 Begin loop 43 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.632755632Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.644456011Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.644770045Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.644810037Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.644822272Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.644843203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} {"msg":"Crucible 0a831839-ac8e-4b98-a59e-3e180e752598 has session id: 4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645198929Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Upstairs opts: Upstairs UUID: 0a831839-ac8e-4b98-a59e-3e180e752598, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645235563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Crucible stats registered with UUID: 0a831839-ac8e-4b98-a59e-3e180e752598","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645251666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:03.64526483Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645445372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645474489Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645489313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645625304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"task":"crutest"} {"msg":"The guest has requested activation with gen:86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645647224Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} {"msg":"0a831839-ac8e-4b98-a59e-3e180e752598 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.64568107Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645695204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"0","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645709367Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"1","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645723511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"2","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645737275Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645754148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645768862Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"ds_connection connected from Ok(127.0.0.1:59786)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645800368Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"2","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"ds_connection connected from Ok(127.0.0.1:37155)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645853355Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"0","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"ds_connection connected from Ok(127.0.0.1:52968)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.645869208Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"io task","client":"1","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.64655702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"1","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.646611546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"0","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.646655377Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"2","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.646916554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.646958206Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[0]R flush_numbers[0..12]: [170, 166, 168, 169, 166, 166, 166, 147, 168, 150, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647046788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[0]R generation[0..12]: [85, 83, 84, 84, 83, 83, 83, 74, 84, 75, 84, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.64706435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647081043Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[1]R flush_numbers[0..12]: [170, 166, 168, 169, 166, 166, 166, 147, 168, 150, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647096986Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[1]R generation[0..12]: [85, 83, 84, 84, 83, 83, 83, 74, 84, 75, 84, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647112389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647132191Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[2]R flush_numbers[0..12]: [170, 166, 168, 169, 166, 166, 166, 147, 168, 150, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647148294Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[2]R generation[0..12]: [85, 83, 84, 84, 83, 83, 83, 74, 84, 75, 84, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647163697Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.64717972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Max found gen is 86","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:52:03.647193034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Generation requested: 86 >= found:86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647212386Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Next flush: 171","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.64722596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647239324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"":"downstairs","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647253418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647266043Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"0","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647279757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"client":"1","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647293471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 :1544,"client":"2","":"downstairs","session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647311233Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647324787Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"0a831839-ac8e-4b98-a59e-3e180e752598 is now active with session: 4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647339971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647354604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544,"session_id":"4e2bc4bb-5143-4b1e-9bb4-b10dfddd9a49"} {"msg":"The guest has finished waiting for activation with:86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647368049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} {"msg":"Activated sub_volume 0a831839-ac8e-4b98-a59e-3e180e752598","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:03.647379474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 346 len 20480 data: 10 8 6 10 11 02/30 Read block 61 len 32768 03/30 Read block 533 len 36864 04/30 Read block 185 len 20480 05/30 Read block 542 len 4096 06/30 Write block 167 len 16384 data: 6 10 11 12 07/30 Write block 188 len 24576 data: 7 7 8 11 9 11 08/30 Flush 09/30 Write block 410 len 24576 data: 10 11 10 10 8 7 10/30 Flush 11/30 Write block 19 len 32768 data: 4 5 7 7 9 9 8 8 12/30 Read block 424 len 40960 13/30 Read block 60 len 28672 14/30 Read block 196 len 36864 15/30 Read block 563 len 4096 16/30 Write block 366 len 12288 data: 5 6 5 17/30 Write block 463 len 24576 data: 6 8 7 7 8 9 18/30 Read block 80 len 12288 19/30 Read block 115 len 28672 20/30 Read block 108 len 4096 21/30 Read block 372 len 8192 22/30 Read block 326 len 24576 23/30 Write block 79 len 12288 data: 8 9 7 24/30 Write block 315 len 32768 data: 6 6 7 7 10 9 8 5 25/30 Read block 129 len 20480 26/30 Write block 467 len 24576 data: 9 10 9 8 9 6 27/30 Write block 81 len 24576 data: 8 8 9 10 11 10 28/30 Write block 445 len 36864 data: 5 5 4 4 5 6 8 10 10 29/30 Write block 589 len 20480 data: 8 7 7 8 7 30/30 Write block 131 len 8192 data: 6 7 ---------------------------------------------------------------- Crucible gen:86 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 8 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Read 5 Done Done Sent false 1004 Acked Read 1 Done Done Sent false 1005 Acked Write 4 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 8 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 7 Done Done Sent false 1013 Acked Read 9 Done Done Sent false 1014 Acked Read 1 Done Done Sent false 1015 Acked Write 3 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 6 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 6 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Write 9 Sent Sent Sent false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 27 33 Done 27 27 3 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1009)) Some(JobId(1009)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:03.658982905Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1544} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 85 85 85 170 170 170 T T F <--- 1 020-039 83 83 83 166 166 166 T T F <--- 2 040-059 84 84 84 168 168 168 F F F 3 060-079 84 84 84 169 169 169 T T F <--- 4 080-099 83 83 83 166 166 166 T T F <--- 5 100-119 83 83 83 166 166 166 F F F 6 120-139 83 83 83 166 166 166 T T F <--- 7 140-159 74 74 74 147 147 147 F F F 8 160-179 86 86 84 171 171 168 F F F <--- 9 180-199 86 86 75 171 171 150 F F F <--- 10 200-219 84 84 84 168 168 168 F F F 11 220-239 84 84 84 168 168 168 F F F 12 240-259 84 84 84 168 168 168 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 84 84 84 168 168 168 T T F <--- 16 320-339 84 84 84 168 168 168 T T F <--- 17 340-359 86 86 84 171 171 168 F F T <--- 18 360-379 81 81 81 164 164 164 T T F <--- 19 380-399 68 68 68 137 137 137 F F F 20 400-419 86 86 82 172 172 165 F F F <--- 21 420-439 82 82 82 165 165 165 F F F 22 440-459 82 82 82 165 165 165 T T F <--- 23 460-479 82 82 82 165 165 165 T T F <--- 24 480-499 82 82 82 165 165 165 F F F 25 500-519 84 84 84 169 169 169 F F F 26 520-539 84 84 84 169 169 169 F F F 27 540-559 85 85 85 170 170 170 F F F 28 560-579 84 84 84 168 168 168 F F F 29 580-599 84 84 84 167 167 167 T T F <--- Max gen: 86, Max flush: 172 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 43 Verifying data now No change after verify Loop: 43 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 87 87 173 173 173 F F F 1 020-039 87 87 87 173 173 173 F F F 2 040-059 84 84 84 168 168 168 F F F 3 060-079 87 87 87 173 173 173 F F F 4 080-099 87 87 87 173 173 173 F F F 5 100-119 83 83 83 166 166 166 F F F 6 120-139 87 87 87 173 173 173 F F F 7 140-159 74 74 74 147 147 147 F F F 8 160-179 86 86 86 171 171 171 F F F 9 180-199 86 86 86 171 171 171 F F F 10 200-219 84 84 84 168 168 168 F F F 11 220-239 84 84 84 168 168 168 F F F 12 240-259 84 84 84 168 168 168 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 87 87 87 173 173 173 F F F 16 320-339 87 87 87 173 173 173 F F F 17 340-359 86 86 86 171 171 171 F F F 18 360-379 87 87 87 173 173 173 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 86 86 86 172 172 172 F F F 21 420-439 82 82 82 165 165 165 F F F 22 440-459 87 87 87 173 173 173 F F F 23 460-479 87 87 87 173 173 173 F F F 24 480-499 82 82 82 165 165 165 F F F 25 500-519 84 84 84 169 169 169 F F F 26 520-539 84 84 84 169 169 169 F F F 27 540-559 85 85 85 170 170 170 F F F 28 560-579 84 84 84 168 168 168 F F F 29 580-599 87 87 87 173 173 173 F F F Max gen: 87, Max flush: 173 Begin loop 44 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:07.991051607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:08.001859143Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.001921956Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.01236547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.012658533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.012697636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.012709871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.012727713Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} {"msg":"Crucible 74bd3123-7ba6-4e54-a9ff-e1ef598a7ac7 has session id: b2ab5352-b960-4c40-b96d-9daaf404561c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013098882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Upstairs opts: Upstairs UUID: 74bd3123-7ba6-4e54-a9ff-e1ef598a7ac7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013133537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Crucible stats registered with UUID: 74bd3123-7ba6-4e54-a9ff-e1ef598a7ac7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.01314969Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:08.013166563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013337399Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013362908Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013377661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013515792Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"task":"crutest"} {"msg":"The guest has requested activation with gen:88","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013532604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} {"msg":"74bd3123-7ba6-4e54-a9ff-e1ef598a7ac7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013553925Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013567219Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"0","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013581073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"1","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013595007Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"2","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013608821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013623535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013637889Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"ds_connection connected from Ok(127.0.0.1:58853)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013696463Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"2","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63356)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013720313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"0","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37264)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.013735726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"io task","client":"1","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014428086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"2","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014455984Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"1","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014472737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"0","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014820176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014856051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[0]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 166, 173, 147, 171, 171, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014910737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[0]R generation[0..12]: [87, 87, 84, 87, 87, 83, 87, 74, 86, 86, 84, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014928359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014945312Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[1]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 166, 173, 147, 171, 171, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014961035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[1]R generation[0..12]: [87, 87, 84, 87, 87, 83, 87, 74, 86, 86, 84, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014976319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.014992562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[2]R flush_numbers[0..12]: [173, 173, 168, 173, 173, 166, 173, 147, 171, 171, 168, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015020879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[2]R generation[0..12]: [87, 87, 84, 87, 87, 83, 87, 74, 86, 86, 84, 84]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015036293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015052576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Max found gen is 88","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015069028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Generation requested: 88 >= found:88","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015083432Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Next flush: 174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015096976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"All extents match","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015116258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"":"downstairs","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015130402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015143246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"0","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.01515742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"1","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015171754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"client":"2","":"downstairs","session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015186358Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015201711Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"74bd3123-7ba6-4e54-a9ff-e1ef598a7ac7 is now active with session: b2ab5352-b960-4c40-b96d-9daaf404561c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015216454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015229169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552,"session_id":"b2ab5352-b960-4c40-b96d-9daaf404561c"} {"msg":"The guest has finished waiting for activation with:88","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015242743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} {"msg":"Activated sub_volume 74bd3123-7ba6-4e54-a9ff-e1ef598a7ac7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:08.015254138Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 327 len 28672 data: 7 6 5 6 6 6 5 02/30 Read block 588 len 12288 03/30 Read block 174 len 32768 04/30 Flush 05/30 Write block 355 len 40960 data: 9 11 12 11 10 8 10 6 7 7 06/30 Write block 437 len 16384 data: 10 11 11 10 07/30 Read block 497 len 40960 08/30 Flush 09/30 Write block 330 len 24576 data: 7 7 7 6 5 7 10/30 Flush 11/30 Write block 155 len 36864 data: 8 8 6 7 9 12 13 11 8 12/30 Read block 281 len 8192 13/30 Write block 241 len 24576 data: 10 8 9 8 7 6 14/30 Flush 15/30 Write block 446 len 12288 data: 6 5 5 16/30 Write block 6 len 4096 data: 7 17/30 Read block 568 len 4096 18/30 Write block 190 len 36864 data: 9 12 10 12 12 10 11 13 13 19/30 Write block 472 len 36864 data: 7 6 6 5 2 4 7 9 11 20/30 Flush 21/30 Write block 328 len 36864 data: 7 6 8 8 8 7 6 8 9 22/30 Flush 23/30 Write block 234 len 24576 data: 7 7 7 8 8 9 24/30 Write block 36 len 24576 data: 9 7 7 7 8 8 25/30 Read block 342 len 20480 26/30 Write block 218 len 28672 data: 3 4 5 6 6 8 7 27/30 Write block 16 len 4096 data: 8 28/30 Write block 525 len 40960 data: 9 7 8 7 8 7 8 7 7 5 29/30 Write block 414 len 8192 data: 9 8 30/30 Write block 556 len 36864 data: 7 6 7 8 8 8 7 7 6 ---------------------------------------------------------------- Crucible gen:88 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Read 8 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 10 Sent Done Done false 1005 Acked Write 4 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 6 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 9 Sent Done Done false 1011 Acked Read 2 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 3 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 9 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Write 7 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Write 10 Sent Sent Sent false 1028 Acked Write 2 Sent Sent Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 3 3 34 Done 2 27 27 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:08.028693504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1552} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 87 88 88 173 178 178 F T T <--- 1 020-039 87 87 87 173 173 173 F T T <--- 2 040-059 84 84 84 168 168 168 F T T <--- 3 060-079 87 87 87 173 173 173 F F F 4 080-099 87 87 87 173 173 173 F F F 5 100-119 83 83 83 166 166 166 F F F 6 120-139 87 87 87 173 173 173 F F F 7 140-159 74 88 88 147 177 177 F F F <--- 8 160-179 86 88 88 171 177 177 F F F <--- 9 180-199 86 88 88 171 178 178 F F F <--- 10 200-219 84 84 84 168 168 168 F T T <--- 11 220-239 84 84 84 168 168 168 F T T <--- 12 240-259 84 88 88 168 177 177 F F F <--- 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 87 87 87 173 173 173 F F F 16 320-339 87 88 88 173 179 179 T F F <--- 17 340-359 86 88 88 171 175 175 F F F <--- 18 360-379 87 88 88 173 175 175 F F F <--- 19 380-399 68 68 68 137 137 137 F F F 20 400-419 86 86 86 172 172 172 F T T <--- 21 420-439 82 88 88 165 175 175 F F F <--- 22 440-459 87 88 88 173 178 178 F F F <--- 23 460-479 87 88 88 173 178 178 F F F <--- 24 480-499 82 88 88 165 178 178 F F F <--- 25 500-519 84 84 84 169 169 169 F F F 26 520-539 84 84 84 169 169 169 F T T <--- 27 540-559 85 85 85 170 170 170 F T T <--- 28 560-579 84 84 84 168 168 168 F T T <--- 29 580-599 87 87 87 173 173 173 F F F Max gen: 88, Max flush: 179 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 44 Verifying data now No change after verify Loop: 44 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 180 180 180 F F F 1 020-039 89 89 89 180 180 180 F F F 2 040-059 89 89 89 180 180 180 F F F 3 060-079 87 87 87 173 173 173 F F F 4 080-099 87 87 87 173 173 173 F F F 5 100-119 83 83 83 166 166 166 F F F 6 120-139 87 87 87 173 173 173 F F F 7 140-159 88 88 88 177 177 177 F F F 8 160-179 88 88 88 177 177 177 F F F 9 180-199 88 88 88 178 178 178 F F F 10 200-219 89 89 89 180 180 180 F F F 11 220-239 89 89 89 180 180 180 F F F 12 240-259 88 88 88 177 177 177 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 87 87 87 173 173 173 F F F 16 320-339 88 88 88 179 179 179 F F F 17 340-359 88 88 88 175 175 175 F F F 18 360-379 88 88 88 175 175 175 F F F 19 380-399 68 68 68 137 137 137 F F F 20 400-419 89 89 89 180 180 180 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 88 88 88 178 178 178 F F F 23 460-479 88 88 88 178 178 178 F F F 24 480-499 88 88 88 178 178 178 F F F 25 500-519 84 84 84 169 169 169 F F F 26 520-539 89 89 89 180 180 180 F F F 27 540-559 89 89 89 180 180 180 F F F 28 560-579 89 89 89 180 180 180 F F F 29 580-599 87 87 87 173 173 173 F F F Max gen: 89, Max flush: 180 Begin loop 45 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.409729226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.421747877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422076884Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422116427Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422128772Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422146494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} {"msg":"Crucible 00b5d320-e1a5-47bb-825b-4035519568ab has session id: 324a4057-dcf0-4dfe-b48a-b68e999275de","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422484258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Upstairs opts: Upstairs UUID: 00b5d320-e1a5-47bb-825b-4035519568ab, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422574109Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Crucible stats registered with UUID: 00b5d320-e1a5-47bb-825b-4035519568ab","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422601127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:12.422615181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.42280272Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422829788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422844742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422976375Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"task":"crutest"} {"msg":"The guest has requested activation with gen:90","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.422995486Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} {"msg":"00b5d320-e1a5-47bb-825b-4035519568ab active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423026453Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423040727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"0","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.42305712Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"1","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423071773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"2","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423085507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423107208Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423121492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"ds_connection connected from Ok(127.0.0.1:43583)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423168171Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"2","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"ds_connection connected from Ok(127.0.0.1:56835)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423184144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"0","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"ds_connection connected from Ok(127.0.0.1:45257)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423199198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"io task","client":"1","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423832024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"1","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423864929Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"0","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.423939467Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"2","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424206042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424246744Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[0]R flush_numbers[0..12]: [180, 180, 180, 173, 173, 166, 173, 177, 177, 178, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424308077Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[0]R generation[0..12]: [89, 89, 89, 87, 87, 83, 87, 88, 88, 88, 89, 89]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.42432507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424344612Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[1]R flush_numbers[0..12]: [180, 180, 180, 173, 173, 166, 173, 177, 177, 178, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424360525Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[1]R generation[0..12]: [89, 89, 89, 87, 87, 83, 87, 88, 88, 88, 89, 89]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424375618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424391951Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[2]R flush_numbers[0..12]: [180, 180, 180, 173, 173, 166, 173, 177, 177, 178, 180, 180]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424407684Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[2]R generation[0..12]: [89, 89, 89, 87, 87, 83, 87, 88, 88, 88, 89, 89]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424422618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424438811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Max found gen is 90","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:52:12.424452105Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Generation requested: 90 >= found:90","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424470267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Next flush: 181","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424483791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424497285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"":"downstairs","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424511139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424523434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"0","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424537188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"1","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Transition from WaitQuorum to Active"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.42455635Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"client":"2","":"downstairs","session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424571383Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424584608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"00b5d320-e1a5-47bb-825b-4035519568ab is now active with session: 324a4057-dcf0-4dfe-b48a-b68e999275de","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424599211Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424611696Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560,"session_id":"324a4057-dcf0-4dfe-b48a-b68e999275de"} {"msg":"The guest has finished waiting for activation with:90","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.42462468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} {"msg":"Activated sub_volume 00b5d320-e1a5-47bb-825b-4035519568ab","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:12.424635995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 221 len 36864 data: 7 7 9 8 4 3 3 5 6 02/30 Read block 557 len 40960 03/30 Write block 202 len 28672 data: 10 11 12 12 9 8 6 04/30 Read block 89 len 16384 05/30 Write block 233 len 24576 data: 8 8 8 8 9 9 06/30 Read block 177 len 28672 07/30 Write block 103 len 20480 data: 8 9 6 6 4 08/30 Read block 572 len 20480 09/30 Write block 199 len 28672 data: 13 12 10 11 12 13 13 10/30 Read block 161 len 12288 11/30 Read block 106 len 4096 12/30 Read block 497 len 40960 13/30 Write block 382 len 28672 data: 7 6 5 6 7 6 7 14/30 Write block 74 len 16384 data: 7 8 7 6 15/30 Read block 254 len 8192 16/30 Read block 342 len 24576 17/30 Write block 505 len 28672 data: 9 11 11 10 9 10 12 18/30 Flush 19/30 Write block 451 len 24576 data: 9 11 11 10 9 10 20/30 Write block 536 len 40960 data: 4 3 3 4 4 4 6 5 5 6 21/30 Read block 279 len 8192 22/30 Read block 237 len 16384 23/30 Write block 62 len 24576 data: 5 7 7 6 6 6 24/30 Read block 52 len 32768 25/30 Write block 330 len 4096 data: 9 26/30 Read block 373 len 12288 27/30 Read block 45 len 28672 28/30 Write block 197 len 8192 data: 14 14 29/30 Read block 447 len 16384 30/30 Read block 560 len 8192 ---------------------------------------------------------------- Crucible gen:90 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Read 10 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Read 4 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Write 7 Done Done Sent false 1009 Acked Read 3 Done Done Sent false 1010 Acked Read 1 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Read 6 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Read 2 Done Done Sent false 1021 Acked Read 4 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Read 8 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:12.437953963Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1560} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 180 180 180 F F F 1 020-039 89 89 89 180 180 180 F F F 2 040-059 89 89 89 180 180 180 F F F 3 060-079 90 90 87 181 181 173 T T F <--- 4 080-099 87 87 87 173 173 173 F F F 5 100-119 90 90 83 181 181 166 F F F <--- 6 120-139 87 87 87 173 173 173 F F F 7 140-159 88 88 88 177 177 177 F F F 8 160-179 88 88 88 177 177 177 F F F 9 180-199 90 90 88 181 181 178 T T F <--- 10 200-219 90 90 89 181 181 180 F F F <--- 11 220-239 90 90 89 181 181 180 F F T <--- 12 240-259 88 88 88 177 177 177 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 87 87 87 173 173 173 F F F 16 320-339 88 88 88 179 179 179 T T F <--- 17 340-359 88 88 88 175 175 175 F F F 18 360-379 88 88 88 175 175 175 F F F 19 380-399 90 90 68 181 181 137 F F F <--- 20 400-419 89 89 89 180 180 180 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 88 88 88 178 178 178 T T F <--- 23 460-479 88 88 88 178 178 178 F F F 24 480-499 88 88 88 178 178 178 F F F 25 500-519 90 90 84 181 181 169 F F F <--- 26 520-539 89 89 89 180 180 180 T T F <--- 27 540-559 89 89 89 180 180 180 T T F <--- 28 560-579 89 89 89 180 180 180 F F F 29 580-599 87 87 87 173 173 173 F F F Max gen: 90, Max flush: 181 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 45 Verifying data now No change after verify Loop: 45 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 180 180 180 F F F 1 020-039 89 89 89 180 180 180 F F F 2 040-059 89 89 89 180 180 180 F F F 3 060-079 91 91 91 182 182 182 F F F 4 080-099 87 87 87 173 173 173 F F F 5 100-119 90 90 90 181 181 181 F F F 6 120-139 87 87 87 173 173 173 F F F 7 140-159 88 88 88 177 177 177 F F F 8 160-179 88 88 88 177 177 177 F F F 9 180-199 91 91 91 182 182 182 F F F 10 200-219 90 90 90 181 181 181 F F F 11 220-239 90 90 90 181 181 181 F F F 12 240-259 88 88 88 177 177 177 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 87 87 87 173 173 173 F F F 16 320-339 91 91 91 182 182 182 F F F 17 340-359 88 88 88 175 175 175 F F F 18 360-379 88 88 88 175 175 175 F F F 19 380-399 90 90 90 181 181 181 F F F 20 400-419 89 89 89 180 180 180 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 91 91 91 182 182 182 F F F 23 460-479 88 88 88 178 178 178 F F F 24 480-499 88 88 88 178 178 178 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 91 91 91 182 182 182 F F F 28 560-579 89 89 89 180 180 180 F F F 29 580-599 87 87 87 173 173 173 F F F Max gen: 91, Max flush: 182 Begin loop 46 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.758742146Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:16.769423507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.769481382Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.779738207Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780045883Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780086196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780098101Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780116083Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} {"msg":"Crucible 5fcfcca0-6b6e-496f-9bd9-f6cbf969fc30 has session id: be3209b3-f24f-4626-87e0-afdf38f03b44","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780482604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Upstairs opts: Upstairs UUID: 5fcfcca0-6b6e-496f-9bd9-f6cbf969fc30, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780517699Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Crucible stats registered with UUID: 5fcfcca0-6b6e-496f-9bd9-f6cbf969fc30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780533572Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:16.780546896Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.78072352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780759754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780775497Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780899194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"task":"crutest"} {"msg":"The guest has requested activation with gen:92","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780916646Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} {"msg":"5fcfcca0-6b6e-496f-9bd9-f6cbf969fc30 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780934308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780947333Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"0","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780961626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"1","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.78097558Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"2","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.780989444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781003998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781018212Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"ds_connection connected from Ok(127.0.0.1:35164)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781035384Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"0","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"ds_connection connected from Ok(127.0.0.1:58606)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781050598Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"2","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"ds_connection connected from Ok(127.0.0.1:33891)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781065581Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"io task","client":"1","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781797224Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"1","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781825682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"2","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.781842465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"0","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782173061Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.78222244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[0]R flush_numbers[0..12]: [180, 180, 180, 182, 173, 181, 173, 177, 177, 182, 181, 181]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782259434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[0]R generation[0..12]: [89, 89, 89, 91, 87, 90, 87, 88, 88, 91, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782276766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782293369Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[1]R flush_numbers[0..12]: [180, 180, 180, 182, 173, 181, 173, 177, 177, 182, 181, 181]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782308973Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[1]R generation[0..12]: [89, 89, 89, 91, 87, 90, 87, 88, 88, 91, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782324286Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782340879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[2]R flush_numbers[0..12]: [180, 180, 180, 182, 173, 181, 173, 177, 177, 182, 181, 181]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782359391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[2]R generation[0..12]: [89, 89, 89, 91, 87, 90, 87, 88, 88, 91, 90, 90]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782374654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782390747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Max found gen is 92","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782404151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"Wait for a query_work_queue command to finish before sending IO downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Generation requested: 92 >= found:92","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782423933Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Next flush: 183","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782437357Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782450691Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"":"downstairs","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782464805Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782477769Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"0","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782491953Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"1","":"downstairs","session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782506047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.7825231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782536654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"5fcfcca0-6b6e-496f-9bd9-f6cbf969fc30 is now active with session: be3209b3-f24f-4626-87e0-afdf38f03b44","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782551647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782564422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568,"session_id":"be3209b3-f24f-4626-87e0-afdf38f03b44"} {"msg":"The guest has finished waiting for activation with:92","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.782580615Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} {"msg":"Activated sub_volume 5fcfcca0-6b6e-496f-9bd9-f6cbf969fc30","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:16.78259223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 405 len 8192 data: 11 11 02/30 Write block 192 len 32768 data: 11 13 13 11 12 15 15 14 03/30 Write block 100 len 24576 data: 11 10 9 9 10 7 04/30 Write block 71 len 28672 data: 4 4 4 8 9 8 7 05/30 Read block 40 len 32768 06/30 Read block 349 len 28672 07/30 Write block 392 len 12288 data: 10 10 7 08/30 Read block 541 len 4096 09/30 Write block 594 len 20480 data: 6 5 4 4 2 10/30 Read block 579 len 4096 11/30 Read block 320 len 32768 12/30 Read block 306 len 4096 13/30 Write block 103 len 36864 data: 10 11 8 7 5 5 5 5 5 14/30 Read block 349 len 16384 15/30 Read block 228 len 40960 16/30 Flush 17/30 Flush 18/30 Write block 44 len 24576 data: 7 7 6 6 6 7 19/30 Read block 92 len 36864 20/30 Read block 267 len 36864 21/30 Write block 176 len 40960 data: 7 6 4 6 6 7 9 9 8 9 22/30 Flush 23/30 Write block 442 len 8192 data: 8 8 24/30 Flush 25/30 Write block 548 len 16384 data: 9 9 10 8 26/30 Write block 303 len 4096 data: 4 27/30 Write block 238 len 8192 data: 10 10 28/30 Write block 93 len 36864 data: 9 8 9 9 10 10 9 12 11 29/30 Read block 146 len 28672 30/30 Write block 121 len 16384 data: 8 10 11 10 ---------------------------------------------------------------- Crucible gen:92 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 6 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 8 Done Done Done false 1005 Acked Read 7 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 1 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 1 Sent Done Done false 1012 Acked Write 9 Sent Done Done false 1013 Acked Read 4 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 6 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 4 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 1 1 27 Done 5 29 29 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:16.794739974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1568} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 180 180 180 F F F 1 020-039 89 89 89 180 180 180 F F F 2 040-059 89 92 92 180 185 185 F F F <--- 3 060-079 91 92 92 182 183 183 T F F <--- 4 080-099 87 87 87 173 173 173 F T T <--- 5 100-119 90 92 92 181 183 183 T T T <--- 6 120-139 87 87 87 173 173 173 F T T <--- 7 140-159 88 88 88 177 177 177 F F F 8 160-179 88 92 92 177 185 185 F F F <--- 9 180-199 91 92 92 182 185 185 T F F <--- 10 200-219 90 90 90 181 181 181 F F F 11 220-239 90 90 90 181 181 181 F T T <--- 12 240-259 88 88 88 177 177 177 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 87 87 87 173 173 173 F T T <--- 16 320-339 91 91 91 182 182 182 F F F 17 340-359 88 88 88 175 175 175 F F F 18 360-379 88 88 88 175 175 175 F F F 19 380-399 90 92 92 181 183 183 F F F <--- 20 400-419 89 92 92 180 183 183 T F F <--- 21 420-439 88 88 88 175 175 175 F F F 22 440-459 91 92 92 182 186 186 F F F <--- 23 460-479 88 88 88 178 178 178 F F F 24 480-499 88 88 88 178 178 178 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 91 91 91 182 182 182 F T T <--- 28 560-579 89 89 89 180 180 180 F F F 29 580-599 87 92 92 173 183 183 F F F <--- Max gen: 92, Max flush: 186 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 46 Verifying data now No change after verify Loop: 46 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 89 89 89 180 180 180 F F F 1 020-039 89 89 89 180 180 180 F F F 2 040-059 92 92 92 185 185 185 F F F 3 060-079 92 92 92 183 183 183 F F F 4 080-099 93 93 93 187 187 187 F F F 5 100-119 93 93 93 187 187 187 F F F 6 120-139 93 93 93 187 187 187 F F F 7 140-159 88 88 88 177 177 177 F F F 8 160-179 92 92 92 185 185 185 F F F 9 180-199 92 92 92 185 185 185 F F F 10 200-219 90 90 90 181 181 181 F F F 11 220-239 93 93 93 187 187 187 F F F 12 240-259 88 88 88 177 177 177 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 93 93 93 187 187 187 F F F 16 320-339 91 91 91 182 182 182 F F F 17 340-359 88 88 88 175 175 175 F F F 18 360-379 88 88 88 175 175 175 F F F 19 380-399 92 92 92 183 183 183 F F F 20 400-419 92 92 92 183 183 183 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 92 92 92 186 186 186 F F F 23 460-479 88 88 88 178 178 178 F F F 24 480-499 88 88 88 178 178 178 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 93 93 93 187 187 187 F F F 28 560-579 89 89 89 180 180 180 F F F 29 580-599 92 92 92 183 183 183 F F F Max gen: 93, Max flush: 187 Begin loop 47 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.131239538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143182252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143497136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.14353407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143546514Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143564476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} {"msg":"Crucible 96d5afcb-cd7b-4230-af55-c247f5ab576f has session id: fa0eb72c-4dfa-4986-9101-e6b4b00e9da3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143911426Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Upstairs opts: Upstairs UUID: 96d5afcb-cd7b-4230-af55-c247f5ab576f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143942662Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Crucible stats registered with UUID: 96d5afcb-cd7b-4230-af55-c247f5ab576f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.143957296Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:21.143970071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144167175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144204409Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144220042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144360641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"task":"crutest"} {"msg":"The guest has requested activation with gen:94","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144375335Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} {"msg":"96d5afcb-cd7b-4230-af55-c247f5ab576f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.14438685Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144399924Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"0","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144413998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"1","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144427892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"2","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144441546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.14445588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144469654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"ds_connection connected from Ok(127.0.0.1:65132)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144531457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"2","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"ds_connection connected from Ok(127.0.0.1:60438)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144646048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"1","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"ds_connection connected from Ok(127.0.0.1:56803)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.144665009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"io task","client":"0","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145276304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"1","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145313288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"0","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145330121Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"2","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145688896Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145705199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[0]R flush_numbers[0..12]: [180, 180, 185, 183, 187, 187, 187, 177, 185, 185, 181, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145738774Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[0]R generation[0..12]: [89, 89, 92, 92, 93, 93, 93, 88, 92, 92, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145755717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145772369Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[1]R flush_numbers[0..12]: [180, 180, 185, 183, 187, 187, 187, 177, 185, 185, 181, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145788113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[1]R generation[0..12]: [89, 89, 92, 92, 93, 93, 93, 88, 92, 92, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145803396Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145819449Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[2]R flush_numbers[0..12]: [180, 180, 185, 183, 187, 187, 187, 177, 185, 185, 181, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145835142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[2]R generation[0..12]: [89, 89, 92, 92, 93, 93, 93, 88, 92, 92, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145849996Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145866179Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Max found gen is 94","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145879513Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Generation requested: 94 >= found:94","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145899215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Next flush: 188","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145912589Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145926013Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"":"downstairs","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145942146Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.14595475Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"0","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145968684Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"1","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145982658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"client":"2","":"downstairs","session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.145997262Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 crucible","level":30,"time":"2026-03-12T22:52:21.146014245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"96d5afcb-cd7b-4230-af55-c247f5ab576f is now active with session: fa0eb72c-4dfa-4986-9101-e6b4b00e9da3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.146029878Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.146042842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576,"session_id":"fa0eb72c-4dfa-4986-9101-e6b4b00e9da3"} {"msg":"The guest has finished waiting for activation with:94","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.146056057Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} {"msg":"Activated sub_volume 96d5afcb-cd7b-4230-af55-c247f5ab576f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:21.146067422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 42 len 40960 data: 7 6 8 8 7 7 7 8 8 9 02/30 Read block 109 len 12288 03/30 Write block 3 len 24576 data: 3 4 6 8 8 10 04/30 Flush 05/30 Write block 42 len 40960 data: 8 7 9 9 8 8 8 9 9 10 06/30 Read block 214 len 28672 07/30 Write block 359 len 40960 data: 11 9 11 7 8 8 5 6 7 6 08/30 Read block 73 len 20480 09/30 Flush 10/30 Write block 557 len 36864 data: 7 8 9 9 9 8 8 7 7 11/30 Write block 479 len 40960 data: 10 12 12 14 12 11 10 9 7 9 12/30 Flush 13/30 Write block 453 len 28672 data: 12 11 10 11 9 6 7 14/30 Write block 151 len 4096 data: 9 15/30 Write block 114 len 36864 data: 6 6 8 6 6 7 8 9 11 16/30 Read block 183 len 28672 17/30 Read block 246 len 40960 18/30 Read block 180 len 8192 19/30 Read block 102 len 40960 20/30 Read block 143 len 4096 21/30 Write block 242 len 40960 data: 9 10 9 8 7 5 5 6 7 7 22/30 Read block 486 len 12288 23/30 Flush 24/30 Write block 157 len 24576 data: 7 8 10 13 14 12 25/30 Read block 524 len 28672 26/30 Write block 179 len 8192 data: 7 7 27/30 Write block 127 len 40960 data: 8 8 7 7 7 8 8 10 9 11 28/30 Read block 187 len 20480 29/30 Write block 126 len 36864 data: 10 9 9 8 8 8 9 9 11 30/30 Write block 65 len 40960 data: 7 7 7 6 5 5 5 5 5 9 ---------------------------------------------------------------- Crucible gen:94 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 6 Done Done Sent false 1003 Acked Flush 0 Done Done Sent false 1004 Acked Write 10 Done Done Sent false 1005 Acked Read 7 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 1 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Read 7 Done Done Sent false 1016 Acked Read 10 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 1 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Read 3 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 7 Done Done Sent false 1025 Acked Write 2 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 28 32 Done 28 28 2 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:21.1606761Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1576} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 89 188 188 180 F F F <--- 1 020-039 89 89 89 180 180 180 F F F 2 040-059 94 94 92 189 189 185 F F T <--- 3 060-079 92 92 92 183 183 183 T T F <--- 4 080-099 93 93 93 187 187 187 F F F 5 100-119 94 94 93 191 191 187 F F F <--- 6 120-139 94 94 93 191 191 187 T T F <--- 7 140-159 94 94 88 191 191 177 T T F <--- 8 160-179 92 92 92 185 185 185 T T F <--- 9 180-199 92 92 92 185 185 185 T T F <--- 10 200-219 90 90 90 181 181 181 F F F 11 220-239 93 93 93 187 187 187 F F F 12 240-259 94 94 88 191 191 177 F F F <--- 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 93 93 93 187 187 187 F F F 16 320-339 91 91 91 182 182 182 F F F 17 340-359 94 94 88 189 189 175 F F F <--- 18 360-379 94 94 88 189 189 175 F F F <--- 19 380-399 92 92 92 183 183 183 F F F 20 400-419 92 92 92 183 183 183 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 94 94 92 191 191 186 F F F <--- 23 460-479 94 94 88 190 190 178 F F F <--- 24 480-499 94 94 88 190 190 178 F F F <--- 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 94 94 93 190 190 187 F F F <--- 28 560-579 94 94 89 190 190 180 F F F <--- 29 580-599 92 92 92 183 183 183 F F F Max gen: 94, Max flush: 191 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 47 Verifying data now No change after verify Loop: 47 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 188 188 188 F F F 1 020-039 89 89 89 180 180 180 F F F 2 040-059 94 94 94 189 189 189 F F F 3 060-079 95 95 95 192 192 192 F F F 4 080-099 93 93 93 187 187 187 F F F 5 100-119 94 94 94 191 191 191 F F F 6 120-139 95 95 95 192 192 192 F F F 7 140-159 95 95 95 192 192 192 F F F 8 160-179 95 95 95 192 192 192 F F F 9 180-199 95 95 95 192 192 192 F F F 10 200-219 90 90 90 181 181 181 F F F 11 220-239 93 93 93 187 187 187 F F F 12 240-259 94 94 94 191 191 191 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 93 93 93 187 187 187 F F F 16 320-339 91 91 91 182 182 182 F F F 17 340-359 94 94 94 189 189 189 F F F 18 360-379 94 94 94 189 189 189 F F F 19 380-399 92 92 92 183 183 183 F F F 20 400-419 92 92 92 183 183 183 F F F 21 420-439 88 88 88 175 175 175 F F F 22 440-459 94 94 94 191 191 191 F F F 23 460-479 94 94 94 190 190 190 F F F 24 480-499 94 94 94 190 190 190 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 94 94 94 190 190 190 F F F 28 560-579 94 94 94 190 190 190 F F F 29 580-599 92 92 92 183 183 183 F F F Max gen: 95, Max flush: 192 Begin loop 48 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.493019384Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:25.503764917Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.503805Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514104216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514396709Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514433104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514448407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514466449Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} {"msg":"Crucible b9c8c4da-bb13-496a-8773-9bf894914023 has session id: 139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514855111Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Upstairs opts: Upstairs UUID: b9c8c4da-bb13-496a-8773-9bf894914023, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514890146Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Crucible stats registered with UUID: b9c8c4da-bb13-496a-8773-9bf894914023","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.514905719Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:25.514919313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.51511064Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515146914Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515162458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515247331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"task":"crutest"} {"msg":"The guest has requested activation with gen:96","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515265213Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} {"msg":"b9c8c4da-bb13-496a-8773-9bf894914023 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515286004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515299028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"0","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515312872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"1","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515329435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"2","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515343459Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515357693Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"ds_connection connected from Ok(127.0.0.1:40489)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515382002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"2","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"ds_connection connected from Ok(127.0.0.1:37533)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515397156Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"1","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.51541127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"ds_connection connected from Ok(127.0.0.1:59894)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.515506079Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"io task","client":"0","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516156157Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"1","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516295976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"2","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516338028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"0","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516548127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516602474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[0]R flush_numbers[0..12]: [188, 180, 189, 192, 187, 191, 192, 192, 192, 192, 181, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516704849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[0]R generation[0..12]: [94, 89, 94, 95, 93, 94, 95, 95, 95, 95, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516721912Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516738065Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[1]R flush_numbers[0..12]: [188, 180, 189, 192, 187, 191, 192, 192, 192, 192, 181, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516756747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[1]R generation[0..12]: [94, 89, 94, 95, 93, 94, 95, 95, 95, 95, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.51677184Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516788153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[2]R flush_numbers[0..12]: [188, 180, 189, 192, 187, 191, 192, 192, 192, 192, 181, 187]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516803946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[2]R generation[0..12]: [94, 89, 94, 95, 93, 94, 95, 95, 95, 95, 90, 93]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.51681899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516835333Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Max found gen is 96","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516848667Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"}Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 96 >= found:96","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516868328Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Next flush: 193","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516881812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516894817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"":"downstairs","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516908691Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516921335Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"0","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516935099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"1","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516948903Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"client":"2","":"downstairs","session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516966866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.5169806Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"b9c8c4da-bb13-496a-8773-9bf894914023 is now active with session: 139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.516995583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.517008288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584,"session_id":"139bfaf0-3b03-4d0b-83d1-6d1f36d8b84c"} {"msg":"The guest has finished waiting for activation with:96","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.517021552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} {"msg":"Activated sub_volume b9c8c4da-bb13-496a-8773-9bf894914023","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:25.517033217Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 154 len 12288 data: 7 9 9 02/30 Read block 372 len 20480 03/30 Write block 132 len 20480 data: 10 10 12 10 12 04/30 Read block 112 len 20480 05/30 Read block 305 len 20480 06/30 Write block 366 len 28672 data: 7 8 7 6 6 4 4 07/30 Read block 170 len 12288 08/30 Flush 09/30 Flush 10/30 Write block 30 len 4096 data: 9 11/30 Read block 330 len 28672 12/30 Write block 55 len 20480 data: 9 8 6 7 7 13/30 Read block 148 len 4096 14/30 Flush 15/30 Write block 413 len 40960 data: 11 10 9 6 7 7 8 7 6 8 16/30 Read block 265 len 32768 17/30 Read block 140 len 16384 18/30 Read block 123 len 32768 19/30 Read block 359 len 8192 20/30 Flush 21/30 Write block 425 len 28672 data: 8 6 3 4 6 6 6 22/30 Write block 385 len 12288 data: 7 8 7 23/30 Read block 66 len 36864 24/30 Read block 98 len 16384 25/30 Read block 286 len 36864 26/30 Read block 44 len 40960 27/30 Read block 198 len 28672 28/30 Write block 181 len 28672 data: 8 10 10 9 10 10 8 29/30 Write block 582 len 20480 data: 8 7 7 8 8 30/30 Write block 222 len 40960 data: 8 10 9 5 4 4 6 7 6 5 ---------------------------------------------------------------- Crucible gen:96 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Write 5 Sent Done Done false 1003 Acked Read 5 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Write 7 Sent Done Done false 1006 Acked Read 3 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 1 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Flush 0 Sent Done Done false 1020 Acked Write 7 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 9 Sent Done Done false 1023 Acked Read 4 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Read 10 Sent Done Done false 1026 Acked Read 7 Sent Done Done false 1027 Acked Write 7 Sent Done Done false 1028 Acked Write 5 Sent Sent Done false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 1 33 Done 0 28 29 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1019)) Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:25.529540006Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1584} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 188 188 188 F F F 1 020-039 89 96 96 180 195 195 F F F <--- 2 040-059 94 96 96 189 195 195 F F F <--- 3 060-079 95 95 95 192 192 192 F F F 4 080-099 93 93 93 187 187 187 F F F 5 100-119 94 94 94 191 191 191 F F F 6 120-139 95 96 96 192 193 193 F F F <--- 7 140-159 95 96 96 192 193 193 F F F <--- 8 160-179 95 95 95 192 192 192 F F F 9 180-199 95 95 95 192 192 192 F T T <--- 10 200-219 90 90 90 181 181 181 F F F 11 220-239 93 93 93 187 187 187 F T T <--- 12 240-259 94 94 94 191 191 191 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 93 93 93 187 187 187 F F F 16 320-339 91 91 91 182 182 182 F F F 17 340-359 94 94 94 189 189 189 F F F 18 360-379 94 96 96 189 193 193 F F F <--- 19 380-399 92 92 92 183 183 183 F T T <--- 20 400-419 92 96 96 183 196 196 F F F <--- 21 420-439 88 96 96 175 196 196 F T T <--- 22 440-459 94 94 94 191 191 191 F F F 23 460-479 94 94 94 190 190 190 F F F 24 480-499 94 94 94 190 190 190 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 94 94 94 190 190 190 F F F 28 560-579 94 94 94 190 190 190 F F F 29 580-599 92 92 92 183 183 183 F T T <--- Max gen: 96, Max flush: 196 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 48 Verifying data now No change after verify Loop: 48 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 94 94 94 188 188 188 F F F 1 020-039 96 96 96 195 195 195 F F F 2 040-059 96 96 96 195 195 195 F F F 3 060-079 95 95 95 192 192 192 F F F 4 080-099 93 93 93 187 187 187 F F F 5 100-119 94 94 94 191 191 191 F F F 6 120-139 96 96 96 193 193 193 F F F 7 140-159 96 96 96 193 193 193 F F F 8 160-179 95 95 95 192 192 192 F F F 9 180-199 97 97 97 197 197 197 F F F 10 200-219 90 90 90 181 181 181 F F F 11 220-239 97 97 97 197 197 197 F F F 12 240-259 94 94 94 191 191 191 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 84 84 84 168 168 168 F F F 15 300-319 93 93 93 187 187 187 F F F 16 320-339 91 91 91 182 182 182 F F F 17 340-359 94 94 94 189 189 189 F F F 18 360-379 96 96 96 193 193 193 F F F 19 380-399 97 97 97 197 197 197 F F F 20 400-419 96 96 96 196 196 196 F F F 21 420-439 97 97 97 197 197 197 F F F 22 440-459 94 94 94 191 191 191 F F F 23 460-479 94 94 94 190 190 190 F F F 24 480-499 94 94 94 190 190 190 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 94 94 94 190 190 190 F F F 28 560-579 94 94 94 190 190 190 F F F 29 580-599 97 97 97 197 197 197 F F F Max gen: 97, Max flush: 197 Begin loop 49 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.85625224Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868156831Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868467417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.86850465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868517295Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868535257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} {"msg":"Crucible ed540393-e32e-494d-bf88-5d3c49ff6d03 has session id: be812b3f-587a-4e0b-8d29-c5da62afef66","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868868423Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Upstairs opts: Upstairs UUID: ed540393-e32e-494d-bf88-5d3c49ff6d03, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868931655Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Crucible stats registered with UUID: ed540393-e32e-494d-bf88-5d3c49ff6d03","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.868948238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:29.868964741Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869133488Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869171112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869187075Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869295458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"task":"crutest"} {"msg":"The guest has requested activation with gen:98","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869311511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} {"msg":"ed540393-e32e-494d-bf88-5d3c49ff6d03 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.86933547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869348625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"0","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869369735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"1","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869383839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"2","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869433888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869458587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.86947364Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"ds_connection connected from Ok(127.0.0.1:57107)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869488824Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"1","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"ds_connection connected from Ok(127.0.0.1:35943)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.869503517Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"2","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"ds_connection connected from Ok(127.0.0.1:41454)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.86961274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"io task","client":"0","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870249414Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"0","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870300312Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"1","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870345832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"2","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870686015Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870737603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[0]R flush_numbers[0..12]: [188, 195, 195, 192, 187, 191, 193, 193, 192, 197, 181, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870778895Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[0]R generation[0..12]: [94, 96, 96, 95, 93, 94, 96, 96, 95, 97, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870795807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.87081252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[1]R flush_numbers[0..12]: [188, 195, 195, 192, 187, 191, 193, 193, 192, 197, 181, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870828773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[1]R generation[0..12]: [94, 96, 96, 95, 93, 94, 96, 96, 95, 97, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870843827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870860179Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[2]R flush_numbers[0..12]: [188, 195, 195, 192, 187, 191, 193, 193, 192, 197, 181, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870876972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[2]R generation[0..12]: [94, 96, 96, 95, 93, 94, 96, 96, 95, 97, 90, 97]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870892555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870909258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Max found gen is 98","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:52:29.870925511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Generation requested: 98 >= found:98","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870946292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Next flush: 198","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870959566Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870972871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"":"downstairs","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870986755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.870999309Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"0","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871013713Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"1","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871029686Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"client":"2","":"downstairs","session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871047138Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871060853Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"ed540393-e32e-494d-bf88-5d3c49ff6d03 is now active with session: be812b3f-587a-4e0b-8d29-c5da62afef66","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871075836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871088561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592,"session_id":"be812b3f-587a-4e0b-8d29-c5da62afef66"} {"msg":"The guest has finished waiting for activation with:98","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.871101655Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} {"msg":"Activated sub_volume ed540393-e32e-494d-bf88-5d3c49ff6d03","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:29.87111297Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 20480 data: 6 5 7 7 9 02/30 Write block 358 len 16384 data: 12 12 10 12 03/30 Read block 569 len 4096 04/30 Read block 406 len 20480 05/30 Write block 3 len 12288 data: 4 5 7 06/30 Read block 72 len 24576 07/30 Write block 383 len 12288 data: 7 6 8 08/30 Write block 316 len 40960 data: 7 8 8 11 10 9 6 6 5 5 09/30 Read block 77 len 20480 10/30 Read block 108 len 8192 11/30 Read block 169 len 20480 12/30 Read block 399 len 12288 13/30 Write block 17 len 28672 data: 7 6 5 6 8 8 10 14/30 Write block 377 len 32768 data: 4 5 5 7 8 8 8 7 15/30 Read block 421 len 24576 16/30 Write block 351 len 40960 data: 10 11 10 10 10 12 13 13 13 11 17/30 Read block 78 len 16384 18/30 Write block 457 len 36864 data: 10 7 8 7 8 6 7 9 8 19/30 Write block 296 len 32768 data: 9 10 8 8 8 8 4 5 20/30 Write block 572 len 20480 data: 7 5 6 5 5 21/30 Write block 332 len 28672 data: 9 8 7 9 10 9 10 22/30 Write block 378 len 24576 data: 6 6 8 9 9 9 23/30 Flush 24/30 Write block 236 len 4096 data: 9 25/30 Read block 255 len 20480 26/30 Flush 27/30 Write block 74 len 32768 data: 10 10 9 8 8 9 10 9 28/30 Write block 489 len 16384 data: 11 7 6 7 29/30 Read block 210 len 40960 30/30 Read block 134 len 28672 ---------------------------------------------------------------- Crucible gen:98 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 4 Done Done Done false 1002 Acked Read 1 Done Done Done false 1003 Acked Read 5 Done Done Done false 1004 Acked Write 3 Done Done Done false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 3 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 2 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Read 3 Done Done Sent false 1012 Acked Write 7 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Write 9 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Write 5 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 4 Done Done Sent false 1028 Acked Read 10 Done Done Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 25 26 Done 30 29 5 64 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:29.883528009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1592} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 94 198 198 188 F F T <--- 1 020-039 98 98 96 198 198 195 F F F <--- 2 040-059 96 96 96 195 195 195 F F F 3 060-079 95 95 95 192 192 192 T T F <--- 4 080-099 93 93 93 187 187 187 T T F <--- 5 100-119 94 94 94 191 191 191 F F F 6 120-139 96 96 96 193 193 193 F F F 7 140-159 96 96 96 193 193 193 F F F 8 160-179 95 95 95 192 192 192 F F F 9 180-199 97 97 97 197 197 197 F F F 10 200-219 90 90 90 181 181 181 F F F 11 220-239 98 98 97 199 199 197 F F F <--- 12 240-259 94 94 94 191 191 191 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 98 98 84 198 198 168 F F F <--- 15 300-319 98 98 93 198 198 187 F F F <--- 16 320-339 98 98 91 198 198 182 F F F <--- 17 340-359 98 98 94 198 198 189 F F T <--- 18 360-379 98 98 96 198 198 193 F F T <--- 19 380-399 98 98 97 198 198 197 F F F <--- 20 400-419 96 96 96 196 196 196 F F F 21 420-439 97 97 97 197 197 197 F F F 22 440-459 98 98 94 198 198 191 F F F <--- 23 460-479 98 98 94 198 198 190 F F F <--- 24 480-499 94 94 94 190 190 190 T T F <--- 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 94 94 94 190 190 190 F F F 28 560-579 98 98 94 198 198 190 F F F <--- 29 580-599 97 97 97 197 197 197 F F F Max gen: 98, Max flush: 199 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 49 Verifying data now No change after verify Loop: 49 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 198 198 198 F F F 1 020-039 98 98 98 198 198 198 F F F 2 040-059 96 96 96 195 195 195 F F F 3 060-079 99 99 99 200 200 200 F F F 4 080-099 99 99 99 200 200 200 F F F 5 100-119 94 94 94 191 191 191 F F F 6 120-139 96 96 96 193 193 193 F F F 7 140-159 96 96 96 193 193 193 F F F 8 160-179 95 95 95 192 192 192 F F F 9 180-199 97 97 97 197 197 197 F F F 10 200-219 90 90 90 181 181 181 F F F 11 220-239 98 98 98 199 199 199 F F F 12 240-259 94 94 94 191 191 191 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 98 98 98 198 198 198 F F F 15 300-319 98 98 98 198 198 198 F F F 16 320-339 98 98 98 198 198 198 F F F 17 340-359 98 98 98 198 198 198 F F F 18 360-379 98 98 98 198 198 198 F F F 19 380-399 98 98 98 198 198 198 F F F 20 400-419 96 96 96 196 196 196 F F F 21 420-439 97 97 97 197 197 197 F F F 22 440-459 98 98 98 198 198 198 F F F 23 460-479 98 98 98 198 198 198 F F F 24 480-499 99 99 99 200 200 200 F F F 25 500-519 90 90 90 181 181 181 F F F 26 520-539 91 91 91 182 182 182 F F F 27 540-559 94 94 94 190 190 190 F F F 28 560-579 98 98 98 198 198 198 F F F 29 580-599 97 97 97 197 197 197 F F F Max gen: 99, Max flush: 200 Begin loop 50 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.209458763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:34.22012553Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.22017284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.230474475Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.230768927Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23080842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.230820605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.230838417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} {"msg":"Crucible ceae98ad-e7fc-4f65-8396-3acd73905334 has session id: 2f2573c6-caeb-4338-ad56-a7b526098f8a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231218372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Upstairs opts: Upstairs UUID: ceae98ad-e7fc-4f65-8396-3acd73905334, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231253747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Crucible stats registered with UUID: ceae98ad-e7fc-4f65-8396-3acd73905334","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23126936Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:34.231282765Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231456339Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231492624Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231507867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231627975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"task":"crutest"} {"msg":"The guest has requested activation with gen:100","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231645947Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} {"msg":"ceae98ad-e7fc-4f65-8396-3acd73905334 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231721175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231735538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"0","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231749882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"1","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231763866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"2","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23177789Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231792144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231806168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:33213)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231840743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"1","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44262)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.231856096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"2","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"ds_connection connected from Ok(127.0.0.1:62192)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23187086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"io task","client":"0","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.232657379Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"2","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.232715504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"1","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.232819888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"0","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233016033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233055166Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[0]R flush_numbers[0..12]: [198, 198, 195, 200, 200, 191, 193, 193, 192, 197, 181, 199]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233190708Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[0]R generation[0..12]: [98, 98, 96, 99, 99, 94, 96, 96, 95, 97, 90, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23320759Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233224243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[1]R flush_numbers[0..12]: [198, 198, 195, 200, 200, 191, 193, 193, 192, 197, 181, 199]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233239946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[1]R generation[0..12]: [98, 98, 96, 99, 99, 94, 96, 96, 95, 97, 90, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23325495Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233271113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[2]R flush_numbers[0..12]: [198, 198, 195, 200, 200, 191, 193, 193, 192, 197, 181, 199]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233286556Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[2]R generation[0..12]: [98, 98, 96, 99, 99, 94, 96, 96, 95, 97, 90, 98]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.23330149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233317643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Max found gen is 100","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233331167ZWait for a query_work_queue command to finish before sending IO ","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Generation requested: 100 >= found:100","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233351658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Next flush: 201","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233365052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233378086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"":"downstairs","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233391851Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233406284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"0","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233420608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"1","":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233437851Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"client":"2","":"downstairs","session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233452104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233465429Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"ceae98ad-e7fc-4f65-8396-3acd73905334 is now active with session: 2f2573c6-caeb-4338-ad56-a7b526098f8a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233480182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233492987Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600,"session_id":"2f2573c6-caeb-4338-ad56-a7b526098f8a"} {"msg":"The guest has finished waiting for activation with:100","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233506181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} {"msg":"Activated sub_volume ceae98ad-e7fc-4f65-8396-3acd73905334","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:34.233517866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 108 len 12288 data: 6 6 6 02/30 Read block 364 len 12288 03/30 Write block 165 len 32768 data: 5 6 7 11 12 13 13 11 04/30 Write block 324 len 36864 data: 6 6 6 8 8 7 10 9 10 05/30 Read block 368 len 40960 06/30 Flush 07/30 Flush 08/30 Write block 249 len 4096 data: 7 09/30 Write block 203 len 32768 data: 13 14 14 10 9 7 8 6 10/30 Write block 319 len 16384 data: 12 11 10 7 11/30 Read block 580 len 8192 12/30 Write block 546 len 12288 data: 5 8 10 13/30 Flush 14/30 Write block 222 len 12288 data: 9 11 10 15/30 Read block 500 len 40960 16/30 Flush 17/30 Write block 514 len 40960 data: 9 9 10 10 11 11 11 9 8 7 18/30 Read block 495 len 40960 19/30 Read block 271 len 12288 20/30 Read block 567 len 28672 21/30 Write block 58 len 20480 data: 8 8 7 5 6 22/30 Read block 411 len 28672 23/30 Read block 487 len 16384 24/30 Read block 366 len 4096 25/30 Flush 26/30 Write block 172 len 16384 data: 12 8 8 8 27/30 Write block 388 len 24576 data: 8 8 8 8 11 11 28/30 Write block 3 len 12288 data: 5 6 8 29/30 Write block 568 len 32768 data: 7 8 7 5 8 6 7 6 30/30 Write block 100 len 28672 data: 13 12 10 11 12 9 8 ---------------------------------------------------------------- Crucible gen:100 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Read 3 Sent Done Done false 1002 Acked Write 8 Sent Done Done false 1003 Acked Write 9 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Flush 0 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Write 4 Sent Done Done false 1010 Acked Read 2 Sent Done Done false 1011 Acked Write 3 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Write 3 Sent Done Done false 1014 Acked Read 10 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Write 10 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 3 Sent Done Done false 1019 Acked Read 7 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 4 Sent Done Done false 1023 Acked Read 1 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 4 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Write 8 Sent Sent Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 2 2 34 Done 0 28 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:34.245743267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1600} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 98 98 98 198 198 198 F T T <--- 1 020-039 98 98 98 198 198 198 F F F 2 040-059 96 100 100 195 205 205 F F F <--- 3 060-079 99 100 100 200 205 205 F F F <--- 4 080-099 99 99 99 200 200 200 F F F 5 100-119 94 100 100 191 201 201 F T T <--- 6 120-139 96 96 96 193 193 193 F F F 7 140-159 96 96 96 193 193 193 F F F 8 160-179 95 100 100 192 201 201 F T T <--- 9 180-199 97 97 97 197 197 197 F F F 10 200-219 90 100 100 181 203 203 F F F <--- 11 220-239 98 100 100 199 204 204 F F F <--- 12 240-259 94 100 100 191 203 203 F F F <--- 13 260-279 84 84 84 168 168 168 F F F 14 280-299 98 98 98 198 198 198 F F F 15 300-319 98 100 100 198 203 203 F F F <--- 16 320-339 98 100 100 198 203 203 F F F <--- 17 340-359 98 98 98 198 198 198 F F F 18 360-379 98 98 98 198 198 198 F F F 19 380-399 98 98 98 198 198 198 F T T <--- 20 400-419 96 96 96 196 196 196 F F F 21 420-439 97 97 97 197 197 197 F F F 22 440-459 98 98 98 198 198 198 F F F 23 460-479 98 98 98 198 198 198 F F F 24 480-499 99 99 99 200 200 200 F F F 25 500-519 90 100 100 181 205 205 F F F <--- 26 520-539 91 100 100 182 205 205 F F F <--- 27 540-559 94 100 100 190 203 203 F F F <--- 28 560-579 98 98 98 198 198 198 F T T <--- 29 580-599 97 97 97 197 197 197 F F F Max gen: 100, Max flush: 205 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 50 Verifying data now No change after verify Loop: 50 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 206 206 206 F F F 1 020-039 98 98 98 198 198 198 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 100 100 100 205 205 205 F F F 4 080-099 99 99 99 200 200 200 F F F 5 100-119 101 101 101 206 206 206 F F F 6 120-139 96 96 96 193 193 193 F F F 7 140-159 96 96 96 193 193 193 F F F 8 160-179 101 101 101 206 206 206 F F F 9 180-199 97 97 97 197 197 197 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 98 98 98 198 198 198 F F F 15 300-319 100 100 100 203 203 203 F F F 16 320-339 100 100 100 203 203 203 F F F 17 340-359 98 98 98 198 198 198 F F F 18 360-379 98 98 98 198 198 198 F F F 19 380-399 101 101 101 206 206 206 F F F 20 400-419 96 96 96 196 196 196 F F F 21 420-439 97 97 97 197 197 197 F F F 22 440-459 98 98 98 198 198 198 F F F 23 460-479 98 98 98 198 198 198 F F F 24 480-499 99 99 99 200 200 200 F F F 25 500-519 100 100 100 205 205 205 F F F 26 520-539 100 100 100 205 205 205 F F F 27 540-559 100 100 100 203 203 203 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 97 97 97 197 197 197 F F F Max gen: 101, Max flush: 206 Begin loop 51 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.583103487Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595007588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595324601Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595363384Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595375958Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595393661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} {"msg":"Crucible bfdd0996-f71c-43d1-9fe5-1d5353d446c9 has session id: a0e6bfac-68cc-40ad-85bf-5008ee12c0e7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595778744Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Upstairs opts: Upstairs UUID: bfdd0996-f71c-43d1-9fe5-1d5353d446c9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595816328Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Crucible stats registered with UUID: bfdd0996-f71c-43d1-9fe5-1d5353d446c9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.595832571Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:38.595846215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596010943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596037862Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596052595Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.59620267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"task":"crutest"} {"msg":"The guest has requested activation with gen:102","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596219503Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} {"msg":"bfdd0996-f71c-43d1-9fe5-1d5353d446c9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596236535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.59624926Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"0","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596262924Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"1","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596276668Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"2","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596290212Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596304246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.59631849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:41587)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596370747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"2","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:43783)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596402554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"0","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"ds_connection connected from Ok(127.0.0.1:60917)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.596486717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"io task","client":"1","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597139444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"0","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597188273Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"2","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597252505Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"1","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597496839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597535383Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[0]R flush_numbers[0..12]: [206, 198, 205, 205, 200, 206, 193, 193, 206, 197, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597640477Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[0]R generation[0..12]: [101, 98, 100, 100, 99, 101, 96, 96, 101, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597657899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597674402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[1]R flush_numbers[0..12]: [206, 198, 205, 205, 200, 206, 193, 193, 206, 197, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597690165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[1]R generation[0..12]: [101, 98, 100, 100, 99, 101, 96, 96, 101, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597705589Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597721802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[2]R flush_numbers[0..12]: [206, 198, 205, 205, 200, 206, 193, 193, 206, 197, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597740754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[2]R generation[0..12]: [101, 98, 100, 100, 99, 101, 96, 96, 101, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597756307Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.59777235Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Max found gen is 102","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:52:38.597786094Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Generation requested: 102 >= found:102","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597804656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Next flush: 207","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.59781798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597830974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"":"downstairs","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597845238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597857993Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"0","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597872127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"1","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597889289Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"client":"2","":"downstairs","session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597903933Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597917137Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"bfdd0996-f71c-43d1-9fe5-1d5353d446c9 is now active with session: a0e6bfac-68cc-40ad-85bf-5008ee12c0e7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597931861Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597944515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608,"session_id":"a0e6bfac-68cc-40ad-85bf-5008ee12c0e7"} {"msg":"The guest has finished waiting for activation with:102","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597959439Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} {"msg":"Activated sub_volume bfdd0996-f71c-43d1-9fe5-1d5353d446c9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:38.597970904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 343 len 36864 data: 10 11 10 11 9 7 11 12 11 02/30 Write block 292 len 32768 data: 10 10 9 9 10 11 9 9 03/30 Read block 19 len 40960 04/30 Write block 144 len 28672 data: 7 7 5 6 9 8 8 05/30 Read block 11 len 36864 06/30 Read block 276 len 32768 07/30 Write block 589 len 32768 data: 9 8 8 9 8 7 6 5 08/30 Read block 389 len 12288 09/30 Flush 10/30 Write block 34 len 12288 data: 9 7 10 11/30 Write block 528 len 20480 data: 8 9 8 9 8 12/30 Write block 301 len 12288 data: 9 5 6 13/30 Read block 545 len 4096 14/30 Flush 15/30 Write block 416 len 24576 data: 7 8 8 9 8 7 16/30 Flush 17/30 Flush 18/30 Write block 100 len 12288 data: 14 13 11 19/30 Flush 20/30 Write block 74 len 28672 data: 11 11 10 9 9 10 11 21/30 Read block 344 len 28672 22/30 Read block 478 len 8192 23/30 Read block 217 len 40960 24/30 Write block 358 len 36864 data: 14 14 12 13 8 9 9 6 8 25/30 Write block 593 len 28672 data: 9 8 7 6 5 3 2 26/30 Read block 417 len 32768 27/30 Read block 419 len 4096 28/30 Write block 422 len 12288 data: 9 8 8 29/30 Write block 428 len 16384 data: 5 7 7 7 30/30 Write block 7 len 24576 data: 9 11 9 9 8 7 ---------------------------------------------------------------- Crucible gen:102 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 9 Done Sent Done false 1005 Acked Read 8 Done Sent Done false 1006 Acked Write 8 Done Sent Done false 1007 Acked Read 3 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 3 Done Sent Done false 1010 Acked Write 5 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 1 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 6 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 3 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 7 Done Sent Done false 1020 Acked Read 7 Done Sent Done false 1021 Acked Read 2 Done Sent Done false 1022 Acked Read 10 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Write 3 Done Sent Done false 1028 Acked Write 4 Done Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 2 30 Done 29 3 28 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:38.610598041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1608} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 101 101 101 206 206 206 T F T <--- 1 020-039 102 98 102 208 198 208 F F F <--- 2 040-059 100 100 100 205 205 205 F F F 3 060-079 100 100 100 205 205 205 T F T <--- 4 080-099 99 99 99 200 200 200 T F T <--- 5 100-119 102 101 102 211 206 211 F F F <--- 6 120-139 96 96 96 193 193 193 F F F 7 140-159 102 96 102 207 193 207 F F F <--- 8 160-179 101 101 101 206 206 206 F F F 9 180-199 97 97 97 197 197 197 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 102 98 102 207 198 207 F T F <--- 15 300-319 102 100 102 208 203 208 F F F <--- 16 320-339 100 100 100 203 203 203 F F F 17 340-359 102 98 102 207 198 207 T T T <--- 18 360-379 98 98 98 198 198 198 T F T <--- 19 380-399 101 101 101 206 206 206 F F F 20 400-419 102 96 102 209 196 209 F F F <--- 21 420-439 102 97 102 209 197 209 T F T <--- 22 440-459 98 98 98 198 198 198 F F F 23 460-479 98 98 98 198 198 198 F F F 24 480-499 99 99 99 200 200 200 F F F 25 500-519 100 100 100 205 205 205 F F F 26 520-539 102 100 102 208 205 208 F F F <--- 27 540-559 100 100 100 203 203 203 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 102 97 102 207 197 207 T F T <--- Max gen: 102, Max flush: 211 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 51 Verifying data now No change after verify Loop: 51 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 212 212 212 F F F 1 020-039 102 102 102 208 208 208 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 103 103 103 212 212 212 F F F 4 080-099 103 103 103 212 212 212 F F F 5 100-119 102 102 102 211 211 211 F F F 6 120-139 96 96 96 193 193 193 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 101 101 101 206 206 206 F F F 9 180-199 97 97 97 197 197 197 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 100 100 100 203 203 203 F F F 13 260-279 84 84 84 168 168 168 F F F 14 280-299 102 102 102 207 207 207 F F F 15 300-319 102 102 102 208 208 208 F F F 16 320-339 100 100 100 203 203 203 F F F 17 340-359 103 103 103 212 212 212 F F F 18 360-379 103 103 103 212 212 212 F F F 19 380-399 101 101 101 206 206 206 F F F 20 400-419 102 102 102 209 209 209 F F F 21 420-439 103 103 103 212 212 212 F F F 22 440-459 98 98 98 198 198 198 F F F 23 460-479 98 98 98 198 198 198 F F F 24 480-499 99 99 99 200 200 200 F F F 25 500-519 100 100 100 205 205 205 F F F 26 520-539 102 102 102 208 208 208 F F F 27 540-559 100 100 100 203 203 203 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 103 103 103 212 212 212 F F F Max gen: 103, Max flush: 212 Begin loop 52 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.936502526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.948253114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.948564019Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.948601723Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.948613927Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.94863161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} {"msg":"Crucible e37f648d-eea8-434f-b109-e6f6f5c2e242 has session id: da8e90bd-ee81-4656-afb2-62ffa583c9c2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.948961057Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Upstairs opts: Upstairs UUID: e37f648d-eea8-434f-b109-e6f6f5c2e242, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949001129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Crucible stats registered with UUID: e37f648d-eea8-434f-b109-e6f6f5c2e242","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949017262Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:42.949030687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949219365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949244254Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949258968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949370939Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"task":"crutest"} {"msg":"The guest has requested activation with gen:104","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949388022Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} {"msg":"e37f648d-eea8-434f-b109-e6f6f5c2e242 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949404485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949417269Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"0","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949431223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"1","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949444987Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"2","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949458891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949473065Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949487119Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:48170)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949501992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"0","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:63107)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949521344Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"2","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53146)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.949634865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"io task","client":"1","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950156319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"1","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950195702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"0","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950508176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950534565Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"2","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950594999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[0]R flush_numbers[0..12]: [212, 208, 205, 212, 212, 211, 193, 207, 206, 197, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950944947Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[0]R generation[0..12]: [103, 102, 100, 103, 103, 102, 96, 102, 101, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950962439Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950979012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[1]R flush_numbers[0..12]: [212, 208, 205, 212, 212, 211, 193, 207, 206, 197, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.950995005Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[1]R generation[0..12]: [103, 102, 100, 103, 103, 102, 96, 102, 101, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951010269Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951026492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[2]R flush_numbers[0..12]: [212, 208, 205, 212, 212, 211, 193, 207, 206, 197, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951042255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[2]R generation[0..12]: [103, 102, 100, 103, 103, 102, 96, 102, 101, 97, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951057968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951074301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Max found gen is 104","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951088015Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Generation requested: 104 >= found:104","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:52:42.951102519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Next flush: 213","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.95112145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951134935Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"":"downstairs","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951151018Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951163972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"0","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951177976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"1","":"downstairs","session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.95119214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"client":"2","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951209562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951223216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"e37f648d-eea8-434f-b109-e6f6f5c2e242 is now active with session: da8e90bd-ee81-4656-afb2-62ffa583c9c2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.95123819Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951251094Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616,"session_id":"da8e90bd-ee81-4656-afb2-62ffa583c9c2"} {"msg":"The guest has finished waiting for activation with:104","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951264568Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} {"msg":"Activated sub_volume e37f648d-eea8-434f-b109-e6f6f5c2e242","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:42.951275974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 285 len 16384 data: 12 15 16 14 03/30 Write block 31 len 28672 data: 9 9 10 10 8 11 8 04/30 Write block 258 len 24576 data: 8 10 12 12 12 9 05/30 Write block 545 len 24576 data: 7 6 9 11 10 11 06/30 Write block 188 len 12288 data: 8 8 10 07/30 Write block 503 len 40960 data: 12 11 10 12 12 11 10 11 13 13 08/30 Write block 323 len 28672 data: 7 7 7 7 9 9 8 09/30 Write block 21 len 36864 data: 9 9 11 10 9 9 8 9 10 10/30 Write block 531 len 8192 data: 10 9 11/30 Read block 295 len 8192 12/30 Write block 366 len 32768 data: 9 9 8 7 7 5 5 2 13/30 Read block 113 len 4096 14/30 Read block 432 len 40960 15/30 Read block 84 len 28672 16/30 Write block 463 len 8192 data: 8 10 17/30 Flush 18/30 Write block 326 len 4096 data: 8 19/30 Read block 119 len 12288 20/30 Write block 135 len 16384 data: 11 13 10 9 21/30 Write block 349 len 28672 data: 12 13 12 12 11 11 11 22/30 Read block 123 len 28672 23/30 Flush 24/30 Write block 266 len 24576 data: 10 9 7 7 6 7 25/30 Read block 371 len 40960 26/30 Flush 27/30 Write block 487 len 32768 data: 8 10 12 8 7 8 8 9 28/30 Read block 513 len 40960 29/30 Write block 384 len 4096 data: 8 30/30 Read block 401 len 8192 ---------------------------------------------------------------- Crucible gen:104 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Done Sent false 1001 Acked Write 4 Done Done Sent false 1002 Acked Write 7 Done Done Sent false 1003 Acked Write 6 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 3 Done Done Sent false 1006 Acked Write 10 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Write 9 Done Done Sent false 1009 Acked Write 2 Done Done Sent false 1010 Acked Read 2 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Read 10 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Flush 0 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 3 Done Done Sent false 1019 Acked Write 4 Done Done Sent false 1020 Acked Write 7 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 10 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Read 10 Done Done Sent false 1028 Acked Write 1 Done Done Sent false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 30 31 Done 30 29 0 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:42.962928943Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1616} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 212 212 212 F F F 1 020-039 104 104 102 214 214 208 F F F <--- 2 040-059 100 100 100 205 205 205 F F F 3 060-079 103 103 103 212 212 212 F F F 4 080-099 103 103 103 212 212 212 F F F 5 100-119 102 102 102 211 211 211 F F F 6 120-139 104 104 96 215 215 193 F F F <--- 7 140-159 102 102 102 207 207 207 F F F 8 160-179 101 101 101 206 206 206 F F F 9 180-199 104 104 97 214 214 197 F F F <--- 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 104 104 100 214 214 203 F F F <--- 13 260-279 104 104 84 216 216 168 F F F <--- 14 280-299 104 104 102 214 214 207 F F F <--- 15 300-319 102 102 102 208 208 208 F F F 16 320-339 104 104 100 215 215 203 F F F <--- 17 340-359 104 104 103 215 215 212 F F F <--- 18 360-379 104 104 103 214 214 212 F F F <--- 19 380-399 101 101 101 206 206 206 T T F <--- 20 400-419 102 102 102 209 209 209 F F F 21 420-439 103 103 103 212 212 212 F F F 22 440-459 98 98 98 198 198 198 F F F 23 460-479 104 104 98 214 214 198 F F F <--- 24 480-499 99 99 99 200 200 200 T T F <--- 25 500-519 104 104 100 214 214 205 F F F <--- 26 520-539 104 104 102 214 214 208 F F F <--- 27 540-559 104 104 100 214 214 203 F F F <--- 28 560-579 101 101 101 206 206 206 F F F 29 580-599 103 103 103 212 212 212 F F F Max gen: 104, Max flush: 216 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 52 Verifying data now No change after verify Loop: 52 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 103 103 103 212 212 212 F F F 1 020-039 104 104 104 214 214 214 F F F 2 040-059 100 100 100 205 205 205 F F F 3 060-079 103 103 103 212 212 212 F F F 4 080-099 103 103 103 212 212 212 F F F 5 100-119 102 102 102 211 211 211 F F F 6 120-139 104 104 104 215 215 215 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 101 101 101 206 206 206 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 104 104 104 214 214 214 F F F 13 260-279 104 104 104 216 216 216 F F F 14 280-299 104 104 104 214 214 214 F F F 15 300-319 102 102 102 208 208 208 F F F 16 320-339 104 104 104 215 215 215 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 105 105 105 217 217 217 F F F 20 400-419 102 102 102 209 209 209 F F F 21 420-439 103 103 103 212 212 212 F F F 22 440-459 98 98 98 198 198 198 F F F 23 460-479 104 104 104 214 214 214 F F F 24 480-499 105 105 105 217 217 217 F F F 25 500-519 104 104 104 214 214 214 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 104 104 104 214 214 214 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 103 103 103 212 212 212 F F F Max gen: 105, Max flush: 217 Begin loop 53 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.286111508Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.2977821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298098583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298140015Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.2981524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298173021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} {"msg":"Crucible c8d97a03-9faa-4272-bb70-7b3204c66fc6 has session id: 5f423baa-0ed8-44d2-9c6c-dc532f99c7e2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.29852159Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Upstairs opts: Upstairs UUID: c8d97a03-9faa-4272-bb70-7b3204c66fc6, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298557704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Crucible stats registered with UUID: c8d97a03-9faa-4272-bb70-7b3204c66fc6","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298573557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:47.298586731Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298757167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298793872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298809355Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.29893685Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"task":"crutest"} {"msg":"The guest has requested activation with gen:106","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298953822Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} {"msg":"c8d97a03-9faa-4272-bb70-7b3204c66fc6 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298977502Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.298990416Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"0","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.29900429Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"1","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299018394Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"2","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299031928Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299049041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299063385Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:60558)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299148858Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"2","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:53881)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.2991665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"1","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"ds_connection connected from Ok(127.0.0.1:59016)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299181484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"io task","client":"0","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.299891646Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"1","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.2999519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"0","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300078835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"2","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300309585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.30034231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[0]R flush_numbers[0..12]: [212, 214, 205, 212, 212, 211, 215, 207, 206, 214, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300448145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[0]R generation[0..12]: [103, 104, 100, 103, 103, 102, 104, 102, 101, 104, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300466037Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.30048257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[1]R flush_numbers[0..12]: [212, 214, 205, 212, 212, 211, 215, 207, 206, 214, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300498093Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[1]R generation[0..12]: [103, 104, 100, 103, 103, 102, 104, 102, 101, 104, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300513756Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300533748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[2]R flush_numbers[0..12]: [212, 214, 205, 212, 212, 211, 215, 207, 206, 214, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.30055014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[2]R generation[0..12]: [103, 104, 100, 103, 103, 102, 104, 102, 101, 104, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300565954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300582396Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 106","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300601998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Generation requested: 106 >= found:106","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300616262Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Next flush: 218","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300630446Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300646149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"":"downstairs","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300660423Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300673127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"0","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300687561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300705233Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"client":"2","":"downstairs","session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300719857Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300733291Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"c8d97a03-9faa-4272-bb70-7b3204c66fc6 is now active with session: 5f423baa-0ed8-44d2-9c6c-dc532f99c7e2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300747985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300762998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624,"session_id":"5f423baa-0ed8-44d2-9c6c-dc532f99c7e2"} {"msg":"The guest has finished waiting for activation with:106","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300776202Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} {"msg":"Activated sub_volume c8d97a03-9faa-4272-bb70-7b3204c66fc6","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:47.300787697Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 4 len 12288 data: 7 9 9 02/30 Write block 507 len 40960 data: 13 12 11 12 14 14 10 10 10 11 03/30 Flush 04/30 Write block 587 len 16384 data: 8 8 10 9 05/30 Read block 562 len 36864 06/30 Read block 355 len 20480 07/30 Write block 139 len 4096 data: 9 08/30 Read block 260 len 4096 09/30 Write block 244 len 4096 data: 10 10/30 Flush 11/30 Write block 398 len 24576 data: 5 5 6 6 6 8 12/30 Flush 13/30 Write block 248 len 24576 data: 6 8 8 8 7 8 14/30 Write block 98 len 36864 data: 11 10 15 14 12 12 13 10 9 15/30 Read block 561 len 12288 16/30 Write block 250 len 40960 data: 9 9 8 9 7 6 6 9 9 11 17/30 Read block 242 len 16384 18/30 Read block 236 len 12288 19/30 Read block 181 len 40960 20/30 Read block 81 len 8192 21/30 Read block 507 len 16384 22/30 Read block 26 len 20480 23/30 Read block 229 len 12288 24/30 Read block 480 len 20480 25/30 Read block 542 len 20480 26/30 Write block 454 len 40960 data: 12 11 12 11 8 9 8 9 7 9 27/30 Read block 457 len 24576 28/30 Write block 59 len 32768 data: 9 8 6 7 8 8 8 8 29/30 Write block 167 len 4096 data: 8 30/30 Write block 328 len 8192 data: 10 9 ---------------------------------------------------------------- Crucible gen:106 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 4 Done Done Done false 1004 Acked Read 9 Done Done Done false 1005 Acked Read 5 Done Done Done false 1006 Acked Write 1 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 1 Done Done Done false 1009 Acked Flush 0 Done Done Sent false 1010 Acked Write 6 Done Done Sent false 1011 Acked Flush 0 Done Done Sent false 1012 Acked Write 6 Done Done Sent false 1013 Acked Write 9 Done Done Sent false 1014 Acked Read 3 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Read 2 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 5 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Write 10 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 8 Sent Sent Sent false 1028 Acked Write 1 Sent Sent Sent false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 3 3 21 27 Done 24 24 6 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1011)) Some(JobId(1011)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:47.312613222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1624} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 218 218 218 F F F 1 020-039 104 104 104 214 214 214 F F F 2 040-059 100 100 100 205 205 205 T T F <--- 3 060-079 103 103 103 212 212 212 T T F <--- 4 080-099 103 103 103 212 212 212 T T F <--- 5 100-119 102 102 102 211 211 211 T T F <--- 6 120-139 106 106 104 219 219 215 F F T <--- 7 140-159 102 102 102 207 207 207 F F F 8 160-179 101 101 101 206 206 206 T T F <--- 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 106 106 104 219 219 214 T T T <--- 13 260-279 104 104 104 216 216 216 F F F 14 280-299 104 104 104 214 214 214 F F F 15 300-319 102 102 102 208 208 208 F F F 16 320-339 104 104 104 215 215 215 T T F <--- 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 106 106 105 220 220 217 F F F <--- 20 400-419 106 106 102 220 220 209 F F F <--- 21 420-439 103 103 103 212 212 212 F F F 22 440-459 98 98 98 198 198 198 T T F <--- 23 460-479 104 104 104 214 214 214 T T F <--- 24 480-499 105 105 105 217 217 217 F F F 25 500-519 106 106 106 218 218 218 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 104 104 104 214 214 214 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 106 106 103 219 219 212 F F T <--- Max gen: 106, Max flush: 220 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 53 Verifying data now No change after verify Loop: 53 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 106 106 106 218 218 218 F F F 1 020-039 104 104 104 214 214 214 F F F 2 040-059 107 107 107 221 221 221 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 107 107 107 221 221 221 F F F 5 100-119 107 107 107 221 221 221 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 100 100 100 204 204 204 F F F 12 240-259 107 107 107 221 221 221 F F F 13 260-279 104 104 104 216 216 216 F F F 14 280-299 104 104 104 214 214 214 F F F 15 300-319 102 102 102 208 208 208 F F F 16 320-339 107 107 107 221 221 221 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 106 106 106 220 220 220 F F F 20 400-419 106 106 106 220 220 220 F F F 21 420-439 103 103 103 212 212 212 F F F 22 440-459 107 107 107 221 221 221 F F F 23 460-479 107 107 107 221 221 221 F F F 24 480-499 105 105 105 217 217 217 F F F 25 500-519 106 106 106 218 218 218 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 104 104 104 214 214 214 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 106 106 106 219 219 219 F F F Max gen: 107, Max flush: 221 Begin loop 54 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.639897249Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.651729302Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652083318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.65212439Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652137205Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652155347Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} {"msg":"Crucible 022a37c9-d2f6-4f95-adae-8cdc07bd1476 has session id: a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652474399Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Upstairs opts: Upstairs UUID: 022a37c9-d2f6-4f95-adae-8cdc07bd1476, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652513182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Crucible stats registered with UUID: 022a37c9-d2f6-4f95-adae-8cdc07bd1476","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652529605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:51.652543349Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652710776Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652735915Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652753348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652887819Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"task":"crutest"} {"msg":"The guest has requested activation with gen:108","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652904872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} {"msg":"022a37c9-d2f6-4f95-adae-8cdc07bd1476 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652927512Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652940407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"0","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.65295476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"1","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652975202Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"2","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.652989016Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653003769Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653017913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:49145)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653066752Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"0","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:33472)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653083405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"2","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"ds_connection connected from Ok(127.0.0.1:48162)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653098548Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"io task","client":"1","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653789539Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"0","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653825303Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"2","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.653841796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"1","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.65415591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.6542022Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[0]R flush_numbers[0..12]: [218, 214, 221, 221, 221, 221, 219, 207, 221, 214, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654237105Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[0]R generation[0..12]: [106, 104, 107, 107, 107, 107, 106, 102, 107, 104, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654254257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.65427056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[1]R flush_numbers[0..12]: [218, 214, 221, 221, 221, 221, 219, 207, 221, 214, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654286543Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[1]R generation[0..12]: [106, 104, 107, 107, 107, 107, 106, 102, 107, 104, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654302396Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654318769Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[2]R flush_numbers[0..12]: [218, 214, 221, 221, 221, 221, 219, 207, 221, 214, 203, 204]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654334542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[2]R generation[0..12]: [106, 104, 107, 107, 107, 107, 106, 102, 107, 104, 100, 100]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654349956Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654366389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Max found gen is 108","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654379763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Generation requested: 108 >= found:108","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654393797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Next flush: 222","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654407241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} Wait for a query_work_queue command to finish before sending IO {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654428422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"":"downstairs","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654442786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.65445545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"0","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654469344Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"1","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654483228Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"client":"2","":"downstairs","session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654497442Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654510876Z","hostname":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"022a37c9-d2f6-4f95-adae-8cdc07bd1476 is now active with session: a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654528678Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654541323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632,"session_id":"a7e5ef1c-52a6-4e17-8465-7b6ec35e78ed"} {"msg":"The guest has finished waiting for activation with:108","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654554787Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} {"msg":"Activated sub_volume 022a37c9-d2f6-4f95-adae-8cdc07bd1476","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:51.654577267Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 110 len 4096 data: 7 02/30 Read block 25 len 4096 03/30 Read block 117 len 12288 04/30 Write block 391 len 8192 data: 9 12 05/30 Read block 520 len 20480 06/30 Write block 468 len 32768 data: 11 10 9 10 8 7 7 6 07/30 Write block 318 len 20480 data: 9 13 12 11 8 08/30 Read block 533 len 4096 09/30 Write block 238 len 8192 data: 11 11 10/30 Read block 363 len 20480 11/30 Write block 38 len 12288 data: 8 8 9 12/30 Read block 440 len 12288 13/30 Write block 443 len 8192 data: 9 7 14/30 Write block 42 len 20480 data: 9 8 10 10 9 15/30 Write block 472 len 16384 data: 9 8 8 7 16/30 Flush 17/30 Write block 35 len 24576 data: 9 12 9 9 9 10 18/30 Flush 19/30 Write block 17 len 12288 data: 8 7 6 20/30 Flush 21/30 Write block 418 len 8192 data: 9 10 22/30 Flush 23/30 Write block 255 len 32768 data: 7 7 10 10 12 13 13 13 24/30 Read block 580 len 28672 25/30 Write block 425 len 12288 data: 9 7 4 26/30 Write block 450 len 32768 data: 7 10 12 13 13 12 13 12 27/30 Read block 328 len 4096 28/30 Read block 168 len 40960 29/30 Read block 336 len 36864 30/30 Read block 519 len 8192 ---------------------------------------------------------------- Crucible gen:108 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 2 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Write 8 Done Done Done false 1006 Acked Write 5 Done Done Done false 1007 Acked Read 1 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Write 3 Done Done Done false 1011 Acked Read 3 Done Done Done false 1012 Acked Write 2 Done Done Done false 1013 Acked Write 5 Done Done Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 6 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 2 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Read 1 Done Sent Done false 1027 Acked Read 10 Done Sent Done false 1028 Acked Read 9 Done Sent Done false 1029 Acked Read 2 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 16 1 17 Done 30 14 29 73 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:51.665944481Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1632} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 106 108 224 218 224 F F F <--- 1 020-039 108 104 108 223 214 223 F T F <--- 2 040-059 108 107 108 223 221 223 F T F <--- 3 060-079 107 107 107 221 221 221 F F F 4 080-099 107 107 107 221 221 221 F F F 5 100-119 108 107 108 222 221 222 F T F <--- 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 108 100 108 222 204 222 F T F <--- 12 240-259 107 107 107 221 221 221 T F T <--- 13 260-279 104 104 104 216 216 216 T F T <--- 14 280-299 104 104 104 214 214 214 F F F 15 300-319 108 102 108 222 208 222 F T F <--- 16 320-339 108 107 108 222 221 222 F T F <--- 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 108 106 108 222 220 222 F T F <--- 20 400-419 108 106 108 225 220 225 F F F <--- 21 420-439 103 103 103 212 212 212 T F T <--- 22 440-459 108 107 108 222 221 222 T T T <--- 23 460-479 108 107 108 222 221 222 F T F <--- 24 480-499 105 105 105 217 217 217 F F F 25 500-519 106 106 106 218 218 218 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 104 104 104 214 214 214 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 106 106 106 219 219 219 F F F Max gen: 108, Max flush: 225 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 54 Verifying data now No change after verify Loop: 54 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 224 224 224 F F F 1 020-039 108 108 108 223 223 223 F F F 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 107 107 107 221 221 221 F F F 5 100-119 108 108 108 222 222 222 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 109 109 109 226 226 226 F F F 13 260-279 109 109 109 226 226 226 F F F 14 280-299 104 104 104 214 214 214 F F F 15 300-319 108 108 108 222 222 222 F F F 16 320-339 108 108 108 222 222 222 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 108 108 108 222 222 222 F F F 20 400-419 108 108 108 225 225 225 F F F 21 420-439 109 109 109 226 226 226 F F F 22 440-459 109 109 109 226 226 226 F F F 23 460-479 108 108 108 222 222 222 F F F 24 480-499 105 105 105 217 217 217 F F F 25 500-519 106 106 106 218 218 218 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 104 104 104 214 214 214 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 106 106 106 219 219 219 F F F Max gen: 109, Max flush: 226 Begin loop 55 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:55.994708907Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:56.005725532Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.005781538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.016300899Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.016593552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.016634224Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.016649008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.01666699Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} {"msg":"Crucible f6ed4736-ffd7-41ad-9eef-9556005f958a has session id: 475212ee-8902-428e-8642-1c5fbce84be9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.016973387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Upstairs opts: Upstairs UUID: f6ed4736-ffd7-41ad-9eef-9556005f958a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017000696Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Crucible stats registered with UUID: f6ed4736-ffd7-41ad-9eef-9556005f958a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017015559Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:56.017028663Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017194032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017218691Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017233255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017374563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"task":"crutest"} {"msg":"The guest has requested activation with gen:110","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017393565Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} {"msg":"f6ed4736-ffd7-41ad-9eef-9556005f958a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017416535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.01742955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"0","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017443873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"1","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017460326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"2","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.01747422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017511014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017533625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"ds_connection connected from Ok(127.0.0.1:42289)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017549458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"1","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"ds_connection connected from Ok(127.0.0.1:61032)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017564561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"2","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"ds_connection connected from Ok(127.0.0.1:44555)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.017638829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"io task","client":"0","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018196657Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"2","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018230672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"1","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018472358Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"0","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.01860509Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018623142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[0]R flush_numbers[0..12]: [224, 223, 223, 221, 221, 222, 219, 207, 221, 214, 203, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018844516Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[0]R generation[0..12]: [108, 108, 108, 107, 107, 108, 106, 102, 107, 104, 100, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018864178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018881201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[1]R flush_numbers[0..12]: [224, 223, 223, 221, 221, 222, 219, 207, 221, 214, 203, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018900032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[1]R generation[0..12]: [108, 108, 108, 107, 107, 108, 106, 102, 107, 104, 100, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018915975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018932128Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[2]R flush_numbers[0..12]: [224, 223, 223, 221, 221, 222, 219, 207, 221, 214, 203, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018947952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[2]R generation[0..12]: [108, 108, 108, 107, 107, 108, 106, 102, 107, 104, 100, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018963865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.018980687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Max found gen is 110","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019012164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Generation requested: 110 >= found:110","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019026608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Next flush: 227","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019039652Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019060103Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"":"downstairs","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019074407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019087111Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"0","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019103364Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"1","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019117898Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"client":"2","":"downstairs","session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019133911Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019147205Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"f6ed4736-ffd7-41ad-9eef-9556005f958a is now active with session: 475212ee-8902-428e-8642-1c5fbce84be9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019162029Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019174933Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640,"session_id":"475212ee-8902-428e-8642-1c5fbce84be9"} {"msg":"The guest has finished waiting for activation with:110","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019188038Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} {"msg":"Activated sub_volume f6ed4736-ffd7-41ad-9eef-9556005f958a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:52:56.019199633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 292 len 28672 data: 11 11 10 10 11 12 10 02/30 Read block 23 len 20480 03/30 Write block 508 len 12288 data: 13 12 13 04/30 Read block 350 len 24576 05/30 Read block 307 len 32768 06/30 Write block 293 len 24576 data: 12 11 11 12 13 11 07/30 Write block 463 len 4096 data: 10 08/30 Read block 70 len 8192 09/30 Read block 328 len 24576 10/30 Read block 426 len 20480 11/30 Flush 12/30 Write block 285 len 32768 data: 13 16 17 15 14 12 14 12 13/30 Write block 541 len 32768 data: 5 7 6 6 8 7 10 12 14/30 Write block 590 len 8192 data: 10 9 15/30 Write block 590 len 40960 data: 11 10 10 10 9 8 7 6 4 3 16/30 Read block 314 len 8192 17/30 Read block 339 len 12288 18/30 Read block 349 len 20480 19/30 Read block 46 len 8192 20/30 Read block 261 len 4096 21/30 Read block 475 len 40960 22/30 Write block 556 len 16384 data: 8 8 9 10 23/30 Read block 412 len 12288 24/30 Read block 375 len 20480 25/30 Read block 182 len 8192 26/30 Read block 298 len 32768 27/30 Read block 436 len 12288 28/30 Read block 48 len 16384 29/30 Read block 335 len 32768 30/30 Read block 314 len 16384 ---------------------------------------------------------------- Crucible gen:110 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Sent Done Done false 1001 Acked Read 5 Sent Done Done false 1002 Acked Write 3 Sent Done Done false 1003 Acked Read 6 Sent Done Done false 1004 Acked Read 8 Sent Done Done false 1005 Acked Write 6 Sent Done Done false 1006 Acked Write 1 Sent Done Done false 1007 Acked Read 2 Sent Done Done false 1008 Acked Read 6 Sent Done Done false 1009 Acked Read 5 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Write 8 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Read 5 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 1 Sent Done Done false 1020 Acked Read 10 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Read 3 Sent Done Done false 1023 Acked Read 5 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 4 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Read 4 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 1 31 Done 0 30 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:52:56.031214515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1640} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 224 224 224 F F F 1 020-039 108 108 108 223 223 223 F F F 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 107 107 107 221 221 221 F F F 5 100-119 108 108 108 222 222 222 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 109 109 109 226 226 226 F F F 13 260-279 109 109 109 226 226 226 F F F 14 280-299 104 110 110 214 227 227 F T T <--- 15 300-319 108 108 108 222 222 222 F F F 16 320-339 108 108 108 222 222 222 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 108 108 108 222 222 222 F F F 20 400-419 108 108 108 225 225 225 F F F 21 420-439 109 109 109 226 226 226 F F F 22 440-459 109 109 109 226 226 226 F F F 23 460-479 108 110 110 222 227 227 F F F <--- 24 480-499 105 105 105 217 217 217 F F F 25 500-519 106 110 110 218 227 227 F F F <--- 26 520-539 104 104 104 214 214 214 F F F 27 540-559 104 104 104 214 214 214 F T T <--- 28 560-579 101 101 101 206 206 206 F F F 29 580-599 106 106 106 219 219 219 F T T <--- Max gen: 110, Max flush: 227 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 55 Verifying data now No change after verify Loop: 55 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 224 224 224 F F F 1 020-039 108 108 108 223 223 223 F F F 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 107 107 107 221 221 221 F F F 5 100-119 108 108 108 222 222 222 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 100 100 203 203 203 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 109 109 109 226 226 226 F F F 13 260-279 109 109 109 226 226 226 F F F 14 280-299 111 111 111 228 228 228 F F F 15 300-319 108 108 108 222 222 222 F F F 16 320-339 108 108 108 222 222 222 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F F F 19 380-399 108 108 108 222 222 222 F F F 20 400-419 108 108 108 225 225 225 F F F 21 420-439 109 109 109 226 226 226 F F F 22 440-459 109 109 109 226 226 226 F F F 23 460-479 110 110 110 227 227 227 F F F 24 480-499 105 105 105 217 217 217 F F F 25 500-519 110 110 110 227 227 227 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 111 111 111 228 228 228 F F F 28 560-579 101 101 101 206 206 206 F F F 29 580-599 111 111 111 228 228 228 F F F Max gen: 111, Max flush: 228 Begin loop 56 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.34383367Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:00.354560301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.354600113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.36483019Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365139516Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365176989Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365189374Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365207306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} {"msg":"Crucible 23f475b2-b53c-4635-8b7d-adbc63d27595 has session id: fff2e260-140d-460f-8a25-75b4f5c2bd6c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365533875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Upstairs opts: Upstairs UUID: 23f475b2-b53c-4635-8b7d-adbc63d27595, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365564871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Crucible stats registered with UUID: 23f475b2-b53c-4635-8b7d-adbc63d27595","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365582204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:00.365595288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.36577497Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365800349Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365815093Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365925805Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"task":"crutest"} {"msg":"The guest has requested activation with gen:112","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365943377Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} {"msg":"23f475b2-b53c-4635-8b7d-adbc63d27595 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365958051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365971025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"0","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365984979Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"1","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.365998673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"2","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366012607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366026841Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366048732Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62428)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366073411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"1","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62845)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366088424Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"2","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"ds_connection connected from Ok(127.0.0.1:44015)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366201775Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"io task","client":"0","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366908789Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"1","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.366958297Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"0","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.36697658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"2","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367276859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367317282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[0]R flush_numbers[0..12]: [224, 223, 223, 221, 221, 222, 219, 207, 221, 214, 203, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367378855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[0]R generation[0..12]: [108, 108, 108, 107, 107, 108, 106, 102, 107, 104, 100, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367395988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.36741275Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[1]R flush_numbers[0..12]: [224, 223, 223, 221, 221, 222, 219, 207, 221, 214, 203, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367428424Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[1]R generation[0..12]: [108, 108, 108, 107, 107, 108, 106, 102, 107, 104, 100, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367444007Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.36746021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[2]R flush_numbers[0..12]: [224, 223, 223, 221, 221, 222, 219, 207, 221, 214, 203, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367475783Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[2]R generation[0..12]: [108, 108, 108, 107, 107, 108, 106, 102, 107, 104, 100, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367494365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367511008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Max found gen is 112","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367524502Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Generation requested: 112 >= found:112","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:53:00.367538576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Next flush: 229","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367556008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367569553Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"":"downstairs","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367583696Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367596441Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"0","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367610565Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"1","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:53:00.367624799Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"client":"2","":"downstairs","session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367640922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367654226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"23f475b2-b53c-4635-8b7d-adbc63d27595 is now active with session: fff2e260-140d-460f-8a25-75b4f5c2bd6c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367669229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367681964Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648,"session_id":"fff2e260-140d-460f-8a25-75b4f5c2bd6c"} {"msg":"The guest has finished waiting for activation with:112","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367695198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} {"msg":"Activated sub_volume 23f475b2-b53c-4635-8b7d-adbc63d27595","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:00.367706513Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 265 len 4096 data: 9 02/30 Read block 11 len 36864 03/30 Read block 543 len 40960 04/30 Read block 239 len 28672 05/30 Read block 387 len 20480 06/30 Read block 523 len 24576 07/30 Write block 478 len 12288 data: 8 11 13 08/30 Read block 505 len 16384 09/30 Flush 10/30 Flush 11/30 Write block 201 len 20480 data: 11 12 14 15 15 12/30 Flush 13/30 Write block 494 len 24576 data: 10 12 14 13 11 13 14/30 Flush 15/30 Write block 270 len 8192 data: 7 8 16/30 Write block 373 len 40960 data: 3 4 4 5 5 7 7 9 10 10 17/30 Read block 194 len 12288 18/30 Write block 87 len 40960 data: 8 7 5 5 5 9 10 9 10 10 19/30 Read block 551 len 28672 20/30 Write block 29 len 32768 data: 11 10 10 10 11 11 10 13 21/30 Read block 526 len 24576 22/30 Write block 255 len 20480 data: 8 8 11 11 13 23/30 Write block 599 len 4096 data: 4 24/30 Read block 447 len 8192 25/30 Read block 355 len 28672 26/30 Read block 451 len 8192 27/30 Read block 222 len 40960 28/30 Read block 403 len 32768 29/30 Write block 554 len 36864 data: 7 5 9 9 10 11 10 10 9 30/30 Read block 310 len 32768 ---------------------------------------------------------------- Crucible gen:112 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Read 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 5 Done Done Done false 1005 Acked Read 6 Sent Done Done false 1006 Acked Write 3 Sent Done Done false 1007 Acked Read 4 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Write 5 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Write 10 Sent Done Done false 1016 Acked Read 3 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 7 Sent Done Done false 1019 Acked Write 8 Sent Done Done false 1020 Acked Read 6 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Read 2 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 2 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 9 Sent Done Done false 1029 Acked Read 8 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 25 0 0 25 Done 5 30 30 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1013)) Some(JobId(1013)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:00.380740784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1648} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 224 224 224 F F F 1 020-039 108 108 108 223 223 223 F T T <--- 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 107 107 107 221 221 221 F T T <--- 5 100-119 108 108 108 222 222 222 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 100 112 112 203 231 231 F F F <--- 11 220-239 108 108 108 222 222 222 F F F 12 240-259 109 109 109 226 226 226 F T T <--- 13 260-279 109 112 112 226 229 229 T T T <--- 14 280-299 111 111 111 228 228 228 F F F 15 300-319 108 108 108 222 222 222 F F F 16 320-339 108 108 108 222 222 222 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 104 104 104 214 214 214 F T T <--- 19 380-399 108 108 108 222 222 222 F T T <--- 20 400-419 108 108 108 225 225 225 F F F 21 420-439 109 109 109 226 226 226 F F F 22 440-459 109 109 109 226 226 226 F F F 23 460-479 110 112 112 227 229 229 F F F <--- 24 480-499 105 112 112 217 232 232 F F F <--- 25 500-519 110 110 110 227 227 227 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 111 111 111 228 228 228 F T T <--- 28 560-579 101 101 101 206 206 206 F T T <--- 29 580-599 111 111 111 228 228 228 F T T <--- Max gen: 112, Max flush: 232 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 56 Verifying data now No change after verify Loop: 56 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 108 108 108 224 224 224 F F F 1 020-039 113 113 113 233 233 233 F F F 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 113 113 113 233 233 233 F F F 5 100-119 108 108 108 222 222 222 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 102 102 102 207 207 207 F F F 8 160-179 107 107 107 221 221 221 F F F 9 180-199 104 104 104 214 214 214 F F F 10 200-219 112 112 112 231 231 231 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 113 113 113 233 233 233 F F F 13 260-279 113 113 113 233 233 233 F F F 14 280-299 111 111 111 228 228 228 F F F 15 300-319 108 108 108 222 222 222 F F F 16 320-339 108 108 108 222 222 222 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 113 113 113 233 233 233 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 108 108 108 225 225 225 F F F 21 420-439 109 109 109 226 226 226 F F F 22 440-459 109 109 109 226 226 226 F F F 23 460-479 112 112 112 229 229 229 F F F 24 480-499 112 112 112 232 232 232 F F F 25 500-519 110 110 110 227 227 227 F F F 26 520-539 104 104 104 214 214 214 F F F 27 540-559 113 113 113 233 233 233 F F F 28 560-579 113 113 113 233 233 233 F F F 29 580-599 113 113 113 233 233 233 F F F Max gen: 113, Max flush: 233 Begin loop 57 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.712697197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.724598759Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.72492061Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.724959773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.724972417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.724990379Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} {"msg":"Crucible 13a90e0e-a4d9-484a-a74f-92a53d29dc55 has session id: 29253e36-6435-42d2-b67a-13042dd17ce2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725329962Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Upstairs opts: Upstairs UUID: 13a90e0e-a4d9-484a-a74f-92a53d29dc55, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725386828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Crucible stats registered with UUID: 13a90e0e-a4d9-484a-a74f-92a53d29dc55","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725414746Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:04.725437886Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.72563571Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725661099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725675722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725798889Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"task":"crutest"} {"msg":"The guest has requested activation with gen:114","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725816571Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} {"msg":"13a90e0e-a4d9-484a-a74f-92a53d29dc55 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725831535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725844809Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"0","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725858593Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"1","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725872707Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"2","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725886401Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725900575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725914689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"ds_connection connected from Ok(127.0.0.1:33087)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725929792Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"2","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"ds_connection connected from Ok(127.0.0.1:51111)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.725944906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"1","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"ds_connection connected from Ok(127.0.0.1:57671)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.7260051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"io task","client":"0","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.726652119Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"2","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.726702607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"0","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.72688095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"1","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727026277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727069149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[0]R flush_numbers[0..12]: [224, 233, 223, 221, 233, 222, 219, 207, 221, 214, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727275489Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[0]R generation[0..12]: [108, 113, 108, 107, 113, 108, 106, 102, 107, 104, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.72729656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727313393Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[1]R flush_numbers[0..12]: [224, 233, 223, 221, 233, 222, 219, 207, 221, 214, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727329046Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[1]R generation[0..12]: [108, 113, 108, 107, 113, 108, 106, 102, 107, 104, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727344609Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727361012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[2]R flush_numbers[0..12]: [224, 233, 223, 221, 233, 222, 219, 207, 221, 214, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727376855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[2]R generation[0..12]: [108, 113, 108, 107, 113, 108, 106, 102, 107, 104, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727392668Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727409141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 114","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727429073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Generation requested: 114 >= found:114","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727443436Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Next flush: 234","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727456781Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727470285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"":"downstairs","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727484209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727497163Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"0","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727514995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"1","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727529779Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"client":"2","":"downstairs","session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727544053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727557817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"13a90e0e-a4d9-484a-a74f-92a53d29dc55 is now active with session: 29253e36-6435-42d2-b67a-13042dd17ce2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.72757296Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727585935Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656,"session_id":"29253e36-6435-42d2-b67a-13042dd17ce2"} {"msg":"The guest has finished waiting for activation with:114","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727599539Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} {"msg":"Activated sub_volume 13a90e0e-a4d9-484a-a74f-92a53d29dc55","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:04.727610934Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 8 len 24576 data: 12 10 10 9 8 6 02/30 Write block 464 len 28672 data: 11 9 8 10 12 11 10 03/30 Read block 152 len 28672 04/30 Read block 228 len 20480 05/30 Read block 389 len 28672 06/30 Write block 320 len 36864 data: 13 12 9 8 8 8 9 10 11 07/30 Read block 174 len 32768 08/30 Write block 532 len 40960 data: 10 8 6 5 5 4 4 5 5 6 09/30 Write block 438 len 32768 data: 12 12 11 10 9 10 8 6 10/30 Write block 160 len 40960 data: 14 15 13 9 6 6 7 9 12 13 11/30 Read block 247 len 20480 12/30 Read block 295 len 40960 13/30 Write block 161 len 16384 data: 16 14 10 7 14/30 Flush 15/30 Write block 151 len 32768 data: 10 6 5 8 10 10 8 9 16/30 Write block 181 len 20480 data: 9 11 11 10 11 17/30 Write block 250 len 12288 data: 10 10 9 18/30 Write block 274 len 16384 data: 6 5 5 5 19/30 Write block 569 len 4096 data: 9 20/30 Write block 526 len 36864 data: 8 9 9 10 9 11 11 9 7 21/30 Write block 12 len 16384 data: 9 7 8 8 22/30 Write block 484 len 32768 data: 12 11 10 9 11 13 9 8 23/30 Write block 92 len 16384 data: 10 11 10 11 24/30 Flush 25/30 Write block 106 len 12288 data: 10 6 7 26/30 Read block 80 len 20480 27/30 Write block 459 len 24576 data: 10 9 10 8 11 12 28/30 Write block 522 len 4096 data: 9 29/30 Read block 31 len 24576 30/30 Write block 404 len 24576 data: 11 12 12 9 9 11 ---------------------------------------------------------------- Crucible gen:114 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 7 Done Sent Done false 1003 Acked Read 5 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 8 Done Sent Done false 1007 Acked Write 10 Done Sent Done false 1008 Acked Write 8 Done Sent Done false 1009 Acked Write 10 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Read 10 Done Sent Done false 1012 Acked Write 4 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 8 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 3 Done Sent Done false 1017 Acked Write 4 Done Sent Done false 1018 Acked Write 1 Done Sent Done false 1019 Acked Write 9 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Write 8 Done Sent Done false 1022 Acked Write 4 Done Sent Done false 1023 Acked Flush 0 Done Sent Done false 1024 Acked Write 3 Done Sent Done false 1025 Acked Read 5 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Read 6 Done Sent Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 28 1 30 Done 29 2 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) None Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:04.739636972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1656} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 108 114 235 224 235 F T F <--- 1 020-039 113 113 113 233 233 233 F F F 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 114 113 114 235 233 235 F F F <--- 5 100-119 108 108 108 222 222 222 T F T <--- 6 120-139 106 106 106 219 219 219 F F F 7 140-159 114 102 114 235 207 235 F F F <--- 8 160-179 114 107 114 234 221 234 F F F <--- 9 180-199 114 104 114 235 214 235 F F F <--- 10 200-219 112 112 112 231 231 231 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 114 113 114 235 233 235 F F F <--- 13 260-279 114 113 114 235 233 235 F F F <--- 14 280-299 111 111 111 228 228 228 F F F 15 300-319 108 108 108 222 222 222 F F F 16 320-339 114 108 114 234 222 234 F F F <--- 17 340-359 104 104 104 215 215 215 F F F 18 360-379 113 113 113 233 233 233 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 108 108 108 225 225 225 T F T <--- 21 420-439 114 109 114 234 226 234 F F F <--- 22 440-459 114 109 114 234 226 234 T F T <--- 23 460-479 114 112 114 234 229 234 T T T <--- 24 480-499 114 112 114 235 232 235 F F F <--- 25 500-519 110 110 110 227 227 227 F F F 26 520-539 114 104 114 235 214 235 T F T <--- 27 540-559 114 113 114 234 233 234 F F F <--- 28 560-579 114 113 114 235 233 235 F F F <--- 29 580-599 113 113 113 233 233 233 F F F Max gen: 114, Max flush: 235 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 57 Verifying data now No change after verify Loop: 57 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 235 235 235 F F F 1 020-039 113 113 113 233 233 233 F F F 2 040-059 108 108 108 223 223 223 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 114 114 114 235 235 235 F F F 5 100-119 115 115 115 236 236 236 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 114 114 114 235 235 235 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 114 114 114 235 235 235 F F F 10 200-219 112 112 112 231 231 231 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 114 114 114 235 235 235 F F F 13 260-279 114 114 114 235 235 235 F F F 14 280-299 111 111 111 228 228 228 F F F 15 300-319 108 108 108 222 222 222 F F F 16 320-339 114 114 114 234 234 234 F F F 17 340-359 104 104 104 215 215 215 F F F 18 360-379 113 113 113 233 233 233 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 114 114 114 234 234 234 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 115 115 115 236 236 236 F F F 24 480-499 114 114 114 235 235 235 F F F 25 500-519 110 110 110 227 227 227 F F F 26 520-539 115 115 115 236 236 236 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 114 114 114 235 235 235 F F F 29 580-599 113 113 113 233 233 233 F F F Max gen: 115, Max flush: 236 Begin loop 58 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.076178887Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:09.086868754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.086919402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.097258751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.097553483Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.097592736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.097608319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.097626192Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} {"msg":"Crucible 711b0dad-8120-48fd-8d7e-1c0cc0834265 has session id: 92c45450-7a64-4bbc-aa62-254143d3698a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.097945103Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Upstairs opts: Upstairs UUID: 711b0dad-8120-48fd-8d7e-1c0cc0834265, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098016782Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Crucible stats registered with UUID: 711b0dad-8120-48fd-8d7e-1c0cc0834265","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098032825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:09.09804612Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098212518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098236757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098251521Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098379005Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"task":"crutest"} {"msg":"The guest has requested activation with gen:116","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098396808Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} {"msg":"711b0dad-8120-48fd-8d7e-1c0cc0834265 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098428944Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098443118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"0","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098457391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"1","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098471445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"2","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.0984849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098501562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098516076Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"ds_connection connected from Ok(127.0.0.1:46988)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098541825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"2","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"ds_connection connected from Ok(127.0.0.1:58606)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098557248Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"1","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"ds_connection connected from Ok(127.0.0.1:38590)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.098572292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"io task","client":"0","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099358001Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"2","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099386419Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"1","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099483897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"0","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099721354Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099761656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[0]R flush_numbers[0..12]: [235, 233, 223, 221, 235, 236, 219, 235, 234, 235, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099888301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[0]R generation[0..12]: [114, 113, 108, 107, 114, 115, 106, 114, 114, 114, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099905824Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099922407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[1]R flush_numbers[0..12]: [235, 233, 223, 221, 235, 236, 219, 235, 234, 235, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099938639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[1]R generation[0..12]: [114, 113, 108, 107, 114, 115, 106, 114, 114, 114, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099957331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099974014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[2]R flush_numbers[0..12]: [235, 233, 223, 221, 235, 236, 219, 235, 234, 235, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.099989817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[2]R generation[0..12]: [114, 113, 108, 107, 114, 115, 106, 114, 114, 114, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100005271Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100021793Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Max found gen is 116","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:53:09.100035428Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Generation requested: 116 >= found:116","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100055569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Next flush: 237","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100068983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100082317Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"":"downstairs","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100096431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100108936Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"0","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.10012296Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"1","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100137053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"client":"2","":"downstairs","session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:53:09.100151287Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100166791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"711b0dad-8120-48fd-8d7e-1c0cc0834265 is now active with session: 92c45450-7a64-4bbc-aa62-254143d3698a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100183233Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100196098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664,"session_id":"92c45450-7a64-4bbc-aa62-254143d3698a"} {"msg":"The guest has finished waiting for activation with:116","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100209542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} {"msg":"Activated sub_volume 711b0dad-8120-48fd-8d7e-1c0cc0834265","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:09.100221117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 44 len 28672 data: 11 11 10 9 9 10 10 02/30 Read block 418 len 28672 03/30 Write block 478 len 28672 data: 9 12 14 13 15 13 13 04/30 Write block 180 len 32768 data: 8 10 12 12 11 12 11 9 05/30 Write block 146 len 20480 data: 6 7 10 9 9 06/30 Write block 529 len 4096 data: 11 07/30 Read block 391 len 16384 08/30 Write block 499 len 4096 data: 14 09/30 Flush 10/30 Write block 297 len 24576 data: 14 12 10 9 10 6 11/30 Read block 362 len 16384 12/30 Write block 486 len 20480 data: 11 10 12 14 10 13/30 Read block 532 len 36864 14/30 Write block 335 len 32768 data: 10 11 10 11 8 7 8 10 15/30 Read block 60 len 8192 16/30 Write block 80 len 16384 data: 12 10 9 10 17/30 Read block 157 len 40960 18/30 Write block 253 len 28672 data: 10 8 9 9 12 12 14 19/30 Write block 565 len 32768 data: 8 6 6 8 10 8 6 9 20/30 Read block 505 len 24576 21/30 Flush 22/30 Write block 261 len 24576 data: 14 14 10 9 10 11 23/30 Read block 86 len 28672 24/30 Read block 30 len 24576 25/30 Write block 29 len 24576 data: 12 11 11 11 12 12 26/30 Write block 321 len 32768 data: 13 10 9 9 9 10 11 12 27/30 Read block 124 len 40960 28/30 Write block 356 len 40960 data: 13 14 15 15 13 14 9 10 10 7 29/30 Read block 63 len 32768 30/30 Write block 185 len 12288 data: 13 12 10 ---------------------------------------------------------------- Crucible gen:116 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Write 7 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Write 5 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 4 Sent Done Done false 1007 Acked Write 1 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Write 5 Sent Done Done false 1012 Acked Read 9 Sent Done Done false 1013 Acked Write 8 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 10 Sent Done Done false 1017 Acked Write 7 Sent Done Done false 1018 Acked Write 8 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 6 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 6 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 8 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 2 29 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1020)) Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:09.114084489Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1664} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 235 235 235 F F F 1 020-039 113 113 113 233 233 233 F T T <--- 2 040-059 108 116 116 223 237 237 T F F <--- 3 060-079 107 107 107 221 221 221 F F F 4 080-099 114 116 116 235 238 238 F F F <--- 5 100-119 115 115 115 236 236 236 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 114 116 116 235 237 237 F F F <--- 8 160-179 114 114 114 234 234 234 F F F 9 180-199 114 116 116 235 237 237 F T T <--- 10 200-219 112 112 112 231 231 231 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 114 116 116 235 238 238 F F F <--- 13 260-279 114 114 114 235 235 235 F T T <--- 14 280-299 111 116 116 228 238 238 F F F <--- 15 300-319 108 116 116 222 238 238 F F F <--- 16 320-339 114 116 116 234 238 238 F T T <--- 17 340-359 104 116 116 215 238 238 F T T <--- 18 360-379 113 113 113 233 233 233 F T T <--- 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 114 114 114 234 234 234 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 115 116 116 236 237 237 F F F <--- 24 480-499 114 116 116 235 238 238 F F F <--- 25 500-519 110 110 110 227 227 227 F F F 26 520-539 115 116 116 236 237 237 F F F <--- 27 540-559 114 114 114 234 234 234 F F F 28 560-579 114 116 116 235 238 238 F F F <--- 29 580-599 113 113 113 233 233 233 F F F Max gen: 116, Max flush: 238 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 58 Verifying data now No change after verify Loop: 58 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 114 114 235 235 235 F F F 1 020-039 117 117 117 239 239 239 F F F 2 040-059 116 116 116 237 237 237 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 116 116 116 238 238 238 F F F 5 100-119 115 115 115 236 236 236 F F F 6 120-139 106 106 106 219 219 219 F F F 7 140-159 116 116 116 237 237 237 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 112 112 112 231 231 231 F F F 11 220-239 108 108 108 222 222 222 F F F 12 240-259 116 116 116 238 238 238 F F F 13 260-279 117 117 117 239 239 239 F F F 14 280-299 116 116 116 238 238 238 F F F 15 300-319 116 116 116 238 238 238 F F F 16 320-339 117 117 117 239 239 239 F F F 17 340-359 117 117 117 239 239 239 F F F 18 360-379 117 117 117 239 239 239 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 114 114 114 234 234 234 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 116 116 116 237 237 237 F F F 24 480-499 116 116 116 238 238 238 F F F 25 500-519 110 110 110 227 227 227 F F F 26 520-539 116 116 116 237 237 237 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 116 116 116 238 238 238 F F F 29 580-599 113 113 113 233 233 233 F F F Max gen: 117, Max flush: 239 Begin loop 59 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.454931288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:13.465691905Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.465740874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.475990601Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476300227Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476349136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476361451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476379003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} {"msg":"Crucible 1659728a-af82-4ac0-8231-b260b06857b8 has session id: 6394b518-3cba-4d43-99de-038c9981696e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476726472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Upstairs opts: Upstairs UUID: 1659728a-af82-4ac0-8231-b260b06857b8, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476762387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Crucible stats registered with UUID: 1659728a-af82-4ac0-8231-b260b06857b8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476781559Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:13.476795023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476958032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476984471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.476998974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477131657Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"task":"crutest"} {"msg":"The guest has requested activation with gen:118","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477149299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} {"msg":"1659728a-af82-4ac0-8231-b260b06857b8 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477165572Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477178217Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"0","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.47719244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"1","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477206544Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"2","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477220488Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477235172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477249386Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"ds_connection connected from Ok(127.0.0.1:53005)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477283961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"2","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61979)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477300484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"0","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"ds_connection connected from Ok(127.0.0.1:60110)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.477317106Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"io task","client":"1","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478089362Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"1","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478118049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"2","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478135122Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"0","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478465059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478482751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[0]R flush_numbers[0..12]: [235, 239, 237, 221, 238, 236, 219, 237, 234, 239, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478517846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[0]R generation[0..12]: [114, 117, 116, 107, 116, 115, 106, 116, 114, 117, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478535119Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478551871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[1]R flush_numbers[0..12]: [235, 239, 237, 221, 238, 236, 219, 237, 234, 239, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478567754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[1]R generation[0..12]: [114, 117, 116, 107, 116, 115, 106, 116, 114, 117, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478583258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478599451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[2]R flush_numbers[0..12]: [235, 239, 237, 221, 238, 236, 219, 237, 234, 239, 231, 222]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478614974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[2]R generation[0..12]: [114, 117, 116, 107, 116, 115, 106, 116, 114, 117, 112, 108]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478630667Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478650129Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Max found gen is 118","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478663813Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Generation requested: 118 >= found:118","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478683234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Next flush: 240","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478696778Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478710253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"":"downstairs","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478724107Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478736701Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"0","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478750735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"1","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:53:13.478764229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"client":"2","":"downstairs","session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478781062Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478794466Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"1659728a-af82-4ac0-8231-b260b06857b8 is now active with session: 6394b518-3cba-4d43-99de-038c9981696e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.4788096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478822514Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672,"session_id":"6394b518-3cba-4d43-99de-038c9981696e"} {"msg":"The guest has finished waiting for activation with:118","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478835568Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} {"msg":"Activated sub_volume 1659728a-af82-4ac0-8231-b260b06857b8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:13.478847183Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 122 len 12288 data: 12 12 11 03/30 Read block 425 len 8192 04/30 Write block 2 len 40960 data: 3 6 8 10 10 10 13 11 11 10 05/30 Write block 23 len 8192 data: 12 11 06/30 Read block 208 len 32768 07/30 Write block 368 len 24576 data: 9 8 8 6 6 4 08/30 Read block 3 len 28672 09/30 Write block 124 len 16384 data: 12 13 11 10 10/30 Read block 348 len 12288 11/30 Flush 12/30 Write block 264 len 24576 data: 10 11 12 10 8 8 13/30 Read block 333 len 4096 14/30 Read block 518 len 36864 15/30 Write block 272 len 32768 data: 8 8 7 6 6 6 4 4 16/30 Read block 17 len 8192 17/30 Write block 144 len 32768 data: 8 8 7 8 11 10 10 11 18/30 Write block 232 len 36864 data: 7 9 9 9 10 10 12 12 10 19/30 Write block 474 len 8192 data: 9 8 20/30 Read block 586 len 20480 21/30 Read block 114 len 12288 22/30 Read block 92 len 36864 23/30 Write block 333 len 16384 data: 9 8 11 12 24/30 Write block 268 len 32768 data: 9 9 8 9 9 9 8 7 25/30 Read block 406 len 40960 26/30 Read block 192 len 4096 27/30 Write block 10 len 32768 data: 12 11 10 8 9 9 10 9 28/30 Read block 97 len 36864 29/30 Write block 370 len 28672 data: 9 7 7 5 5 5 6 30/30 Write block 491 len 36864 data: 9 9 9 11 13 15 14 12 15 ---------------------------------------------------------------- Crucible gen:118 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Sent Done Done false 1002 Acked Read 2 Sent Done Done false 1003 Acked Write 10 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Write 4 Sent Done Done false 1009 Acked Read 3 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 1 Sent Done Done false 1013 Acked Read 9 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Write 8 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Read 9 Sent Done Done false 1022 Acked Write 4 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 10 Sent Done Done false 1025 Acked Read 1 Sent Done Done false 1026 Acked Write 8 Sent Done Done false 1027 Acked Read 9 Sent Done Done false 1028 Acked Write 7 Sent Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 1 2 32 Done 0 28 27 55 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:13.491843331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1672} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 114 118 118 235 241 241 F T T <--- 1 020-039 117 118 118 239 241 241 F F F <--- 2 040-059 116 116 116 237 237 237 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 116 116 116 238 238 238 F F F 5 100-119 115 115 115 236 236 236 F F F 6 120-139 106 118 118 219 241 241 F F F <--- 7 140-159 116 116 116 237 237 237 F T T <--- 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 112 112 112 231 231 231 F F F 11 220-239 108 108 108 222 222 222 F T T <--- 12 240-259 116 116 116 238 238 238 F T T <--- 13 260-279 117 117 117 239 239 239 F T T <--- 14 280-299 116 116 116 238 238 238 F F F 15 300-319 116 116 116 238 238 238 F F F 16 320-339 117 117 117 239 239 239 F T T <--- 17 340-359 117 117 117 239 239 239 F F F 18 360-379 117 118 118 239 241 241 F T T <--- 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 114 114 114 234 234 234 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 116 116 116 237 237 237 F T T <--- 24 480-499 116 116 116 238 238 238 F T T <--- 25 500-519 110 110 110 227 227 227 F F F 26 520-539 116 116 116 237 237 237 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 116 116 116 238 238 238 F F F 29 580-599 113 113 113 233 233 233 F F F Max gen: 118, Max flush: 241 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 59 Verifying data now No change after verify Loop: 59 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 242 242 242 F F F 1 020-039 118 118 118 241 241 241 F F F 2 040-059 116 116 116 237 237 237 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 116 116 116 238 238 238 F F F 5 100-119 115 115 115 236 236 236 F F F 6 120-139 118 118 118 241 241 241 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 112 112 112 231 231 231 F F F 11 220-239 119 119 119 242 242 242 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 119 119 119 242 242 242 F F F 14 280-299 116 116 116 238 238 238 F F F 15 300-319 116 116 116 238 238 238 F F F 16 320-339 119 119 119 242 242 242 F F F 17 340-359 117 117 117 239 239 239 F F F 18 360-379 119 119 119 242 242 242 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 114 114 114 234 234 234 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 119 119 119 242 242 242 F F F 24 480-499 119 119 119 242 242 242 F F F 25 500-519 110 110 110 227 227 227 F F F 26 520-539 116 116 116 237 237 237 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 116 116 116 238 238 238 F F F 29 580-599 113 113 113 233 233 233 F F F Max gen: 119, Max flush: 242 Begin loop 60 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.821773771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.833648604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.833970525Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834010367Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834023102Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834041024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} {"msg":"Crucible 268792d2-a055-47b9-a686-0066e1e2d4f7 has session id: 324a03b5-04bf-4dc4-ac5f-6944cd259bf3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834400618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Upstairs opts: Upstairs UUID: 268792d2-a055-47b9-a686-0066e1e2d4f7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834430215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Crucible stats registered with UUID: 268792d2-a055-47b9-a686-0066e1e2d4f7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834444949Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:17.834458153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834659466Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.8346956Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834711394Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834796207Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"task":"crutest"} {"msg":"The guest has requested activation with gen:120","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} {"msg":"268792d2-a055-47b9-a686-0066e1e2d4f7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834829542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834844746Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"0","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.83485897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"1","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834872634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"2","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834885918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834899882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834913946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:40867)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834948781Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"2","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62730)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834965054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"1","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"ds_connection connected from Ok(127.0.0.1:62170)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.834979897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"io task","client":"0","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.835730732Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"0","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.835767066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"1","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.835783469Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"2","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836115995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836163324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[0]R flush_numbers[0..12]: [242, 241, 237, 221, 238, 236, 241, 242, 234, 239, 231, 242]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836200588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[0]R generation[0..12]: [119, 118, 116, 107, 116, 115, 118, 119, 114, 117, 112, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.83622006Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836236832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[1]R flush_numbers[0..12]: [242, 241, 237, 221, 238, 236, 241, 242, 234, 239, 231, 242]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836252436Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[1]R generation[0..12]: [119, 118, 116, 107, 116, 115, 118, 119, 114, 117, 112, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836267969Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836284002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[2]R flush_numbers[0..12]: [242, 241, 237, 221, 238, 236, 241, 242, 234, 239, 231, 242]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836299755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[2]R generation[0..12]: [119, 118, 116, 107, 116, 115, 118, 119, 114, 117, 112, 119]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836315069Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836331182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Max found gen is 120","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836344596Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Generation requested: 120 >= found:120","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836363877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Next flush: 243","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836377272Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836390576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"":"downstairs","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.83640444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836416904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"0","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836432088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"1","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836446092Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"client":"2","":"downstairs","session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836463314Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836476479Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"268792d2-a055-47b9-a686-0066e1e2d4f7 is now active with session: 324a03b5-04bf-4dc4-ac5f-6944cd259bf3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836491172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836503627Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680,"session_id":"324a03b5-04bf-4dc4-ac5f-6944cd259bf3"} {"msg":"The guest has finished waiting for activation with:120","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836516641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} {"msg":"Activated sub_volume 268792d2-a055-47b9-a686-0066e1e2d4f7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:17.836528116Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 516 len 4096 data: 12 02/30 Write block 227 len 8192 data: 5 7 03/30 Write block 423 len 20480 data: 9 9 10 8 5 04/30 Read block 234 len 8192 05/30 Write block 118 len 28672 data: 7 8 9 10 13 13 13 06/30 Write block 342 len 36864 data: 11 11 12 11 12 10 8 13 14 07/30 Read block 59 len 40960 08/30 Flush 09/30 Write block 311 len 28672 data: 8 8 5 5 7 8 9 10/30 Read block 259 len 40960 11/30 Read block 407 len 20480 12/30 Write block 94 len 32768 data: 11 12 11 11 12 11 16 15 13/30 Read block 517 len 8192 14/30 Write block 278 len 16384 data: 5 5 5 3 15/30 Write block 337 len 12288 data: 11 12 9 16/30 Read block 422 len 28672 17/30 Read block 193 len 16384 18/30 Write block 498 len 8192 data: 13 16 19/30 Flush 20/30 Write block 4 len 20480 data: 9 11 11 11 14 21/30 Read block 322 len 12288 22/30 Write block 427 len 12288 data: 6 6 8 23/30 Write block 562 len 36864 data: 10 9 8 9 7 7 9 11 9 24/30 Write block 584 len 32768 data: 8 9 9 9 9 11 12 11 25/30 Read block 318 len 32768 26/30 Read block 274 len 32768 27/30 Read block 519 len 32768 28/30 Write block 49 len 16384 data: 11 11 11 8 29/30 Read block 507 len 32768 30/30 Write block 202 len 8192 data: 13 15 ---------------------------------------------------------------- Crucible gen:120 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Write 7 Done Sent Done false 1005 Acked Write 9 Done Sent Done false 1006 Acked Read 10 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 7 Done Sent Done false 1009 Acked Read 10 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 8 Done Sent Done false 1012 Acked Read 2 Done Sent Done false 1013 Acked Write 4 Done Sent Done false 1014 Acked Write 3 Done Sent Done false 1015 Acked Read 7 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Write 2 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 5 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 3 Done Sent Done false 1022 Acked Write 9 Done Sent Done false 1023 Acked Write 8 Done Sent Done false 1024 Acked Read 8 Done Sent Done false 1025 Acked Read 8 Done Sent Done false 1026 Acked Read 8 Done Sent Done false 1027 Acked Write 4 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 2 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 27 1 29 Done 29 3 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1018)) None Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:17.848566049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1680} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 119 119 119 242 242 242 T F T <--- 1 020-039 118 118 118 241 241 241 F F F 2 040-059 116 116 116 237 237 237 T F T <--- 3 060-079 107 107 107 221 221 221 F F F 4 080-099 120 116 120 244 238 244 F F F <--- 5 100-119 120 115 120 244 236 244 F F F <--- 6 120-139 120 118 120 243 241 243 F F F <--- 7 140-159 119 119 119 242 242 242 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 112 112 112 231 231 231 T F T <--- 11 220-239 120 119 120 243 242 243 F T F <--- 12 240-259 119 119 119 242 242 242 F F F 13 260-279 120 119 120 244 242 244 F F F <--- 14 280-299 120 116 120 244 238 244 F F F <--- 15 300-319 120 116 120 244 238 244 F F F <--- 16 320-339 120 119 120 244 242 244 F F F <--- 17 340-359 120 117 120 243 239 243 F F F <--- 18 360-379 119 119 119 242 242 242 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 120 114 120 243 234 243 T T T <--- 22 440-459 115 115 115 236 236 236 F F F 23 460-479 119 119 119 242 242 242 F F F 24 480-499 120 119 120 244 242 244 F F F <--- 25 500-519 120 110 120 243 227 243 F T F <--- 26 520-539 116 116 116 237 237 237 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 116 116 116 238 238 238 T F T <--- 29 580-599 113 113 113 233 233 233 T F T <--- Max gen: 120, Max flush: 244 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 60 Verifying data now No change after verify Loop: 60 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 245 245 245 F F F 1 020-039 118 118 118 241 241 241 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 120 120 120 243 243 243 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 121 121 121 245 245 245 F F F 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 120 120 120 244 244 244 F F F 15 300-319 120 120 120 244 244 244 F F F 16 320-339 120 120 120 244 244 244 F F F 17 340-359 120 120 120 243 243 243 F F F 18 360-379 119 119 119 242 242 242 F F F 19 380-399 113 113 113 233 233 233 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 121 121 121 245 245 245 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 119 119 119 242 242 242 F F F 24 480-499 120 120 120 244 244 244 F F F 25 500-519 120 120 120 243 243 243 F F F 26 520-539 116 116 116 237 237 237 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 121 121 121 245 245 245 F F F 29 580-599 121 121 121 245 245 245 F F F Max gen: 121, Max flush: 245 Begin loop 61 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.184979499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.196726658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197040042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197079855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.1970919Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197110122Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} {"msg":"Crucible a530adf7-166d-43fd-8ed0-a1ac0aaab64e has session id: 26226005-306b-4722-9cea-055f460c1d91","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197476333Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Upstairs opts: Upstairs UUID: a530adf7-166d-43fd-8ed0-a1ac0aaab64e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.19752999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Crucible stats registered with UUID: a530adf7-166d-43fd-8ed0-a1ac0aaab64e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197546203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:22.197559557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.19773739Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197774524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197790127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197923289Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"task":"crutest"} {"msg":"The guest has requested activation with gen:122","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197940002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} {"msg":"a530adf7-166d-43fd-8ed0-a1ac0aaab64e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197961443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197974537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"0","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.197988841Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"1","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198002745Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"2","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198016319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198030443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198047056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"ds_connection connected from Ok(127.0.0.1:42105)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198118825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"2","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"ds_connection connected from Ok(127.0.0.1:40993)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198149032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"1","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"ds_connection connected from Ok(127.0.0.1:35500)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198238053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"io task","client":"0","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198818412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"1","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198859414Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"0","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.198980951Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"2","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199186222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199227724Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[0]R flush_numbers[0..12]: [245, 241, 245, 221, 244, 244, 243, 242, 234, 239, 245, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199396231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[0]R generation[0..12]: [121, 118, 121, 107, 120, 120, 120, 119, 114, 117, 121, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199424609Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199442051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[1]R flush_numbers[0..12]: [245, 241, 245, 221, 244, 244, 243, 242, 234, 239, 245, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199458164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[1]R generation[0..12]: [121, 118, 121, 107, 120, 120, 120, 119, 114, 117, 121, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199473578Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199489691Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[2]R flush_numbers[0..12]: [245, 241, 245, 221, 244, 244, 243, 242, 234, 239, 245, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199508283Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[2]R generation[0..12]: [121, 118, 121, 107, 120, 120, 120, 119, 114, 117, 121, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199532702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs",Wait for a query_work_queue command to finish before sending IO "":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199555682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Max found gen is 122","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199569426Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Generation requested: 122 >= found:122","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.1995836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Next flush: 246","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199596934Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199610298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"":"downstairs","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199624402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199636977Z","hostnameDisk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"0","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199654299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"1","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199668553Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"client":"2","":"downstairs","session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199682797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199696391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"a530adf7-166d-43fd-8ed0-a1ac0aaab64e is now active with session: 26226005-306b-4722-9cea-055f460c1d91","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199711135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199723749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688,"session_id":"26226005-306b-4722-9cea-055f460c1d91"} {"msg":"The guest has finished waiting for activation with:122","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199736834Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} {"msg":"Activated sub_volume a530adf7-166d-43fd-8ed0-a1ac0aaab64e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:22.199749968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 341 len 8192 data: 9 12 02/30 Write block 521 len 20480 data: 10 10 8 7 10 03/30 Read block 502 len 28672 04/30 Write block 338 len 12288 data: 13 10 8 05/30 Read block 398 len 24576 06/30 Write block 213 len 28672 data: 7 6 5 4 4 4 5 07/30 Flush 08/30 Write block 490 len 12288 data: 11 10 10 09/30 Write block 324 len 20480 data: 10 10 11 12 13 10/30 Read block 149 len 20480 11/30 Read block 484 len 28672 12/30 Read block 507 len 36864 13/30 Read block 465 len 16384 14/30 Flush 15/30 Flush 16/30 Write block 296 len 40960 data: 13 15 13 11 10 11 7 7 6 6 17/30 Write block 28 len 16384 data: 10 13 12 12 18/30 Read block 303 len 36864 19/30 Write block 282 len 24576 data: 6 5 9 14 17 18 20/30 Read block 384 len 12288 21/30 Read block 37 len 28672 22/30 Read block 130 len 8192 23/30 Write block 381 len 8192 data: 11 11 24/30 Read block 201 len 12288 25/30 Read block 23 len 20480 26/30 Flush 27/30 Write block 14 len 36864 data: 10 10 11 10 8 7 7 10 10 28/30 Read block 594 len 16384 29/30 Read block 356 len 16384 30/30 Read block 268 len 12288 ---------------------------------------------------------------- Crucible gen:122 GIO:true work queues: Upstairs:0 downstairs:23 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1007 Acked Write 3 Done Done Done false 1008 Acked Write 5 Done Done Done false 1009 Acked Read 5 Done Done Done false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Flush 0 Done Done Sent false 1015 Acked Write 10 Done Done Sent false 1016 Acked Write 4 Done Done Sent false 1017 Acked Read 9 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Read 7 Done Done Sent false 1021 Acked Read 2 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 3 Done Done Sent false 1024 Acked Read 5 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 9 Done Done Sent false 1027 Acked Read 4 Done Done Sent false 1028 Acked Read 4 Done Done Sent false 1029 Acked Read 3 Done Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 0 20 20 Done 23 23 3 49 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1006)) Downstairs last five completed: 1006 1005 1004 1003 1002 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:22.211806832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1688} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 121 121 121 245 245 245 T T F <--- 1 020-039 122 122 118 249 249 241 T T F <--- 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 120 120 120 243 243 243 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 122 122 122 246 246 246 F F F 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 122 122 120 249 249 244 F F F <--- 15 300-319 122 122 120 249 249 244 F F F <--- 16 320-339 122 122 122 247 247 246 F F T <--- 17 340-359 122 122 122 246 246 246 F F F 18 360-379 119 119 119 242 242 242 F F F 19 380-399 122 122 113 249 249 233 F F F <--- 20 400-419 115 115 115 236 236 236 F F F 21 420-439 121 121 121 245 245 245 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 119 119 119 242 242 242 F F F 24 480-499 122 122 120 247 247 244 F F T <--- 25 500-519 120 120 120 243 243 243 F F F 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 121 121 121 245 245 245 F F F 29 580-599 121 121 121 245 245 245 F F F Max gen: 122, Max flush: 249 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 61 Verifying data now No change after verify Loop: 61 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 250 250 250 F F F 1 020-039 123 123 123 250 250 250 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 120 120 120 243 243 243 F F F 7 140-159 119 119 119 242 242 242 F F F 8 160-179 114 114 114 234 234 234 F F F 9 180-199 117 117 117 239 239 239 F F F 10 200-219 122 122 122 246 246 246 F F F 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 120 120 120 244 244 244 F F F 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 122 122 122 247 247 247 F F F 17 340-359 122 122 122 246 246 246 F F F 18 360-379 119 119 119 242 242 242 F F F 19 380-399 122 122 122 249 249 249 F F F 20 400-419 115 115 115 236 236 236 F F F 21 420-439 121 121 121 245 245 245 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 119 119 119 242 242 242 F F F 24 480-499 122 122 122 247 247 247 F F F 25 500-519 120 120 120 243 243 243 F F F 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 121 121 121 245 245 245 F F F 29 580-599 121 121 121 245 245 245 F F F Max gen: 123, Max flush: 250 Begin loop 62 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.517756695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.529451457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.529814469Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.529856451Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.529868596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.529886618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} {"msg":"Crucible dbe46b28-d141-459c-9ddd-2f70b6736dba has session id: c85fe171-6fdf-429d-be75-47aeec3eac15","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530216815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Upstairs opts: Upstairs UUID: dbe46b28-d141-459c-9ddd-2f70b6736dba, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530263825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Crucible stats registered with UUID: dbe46b28-d141-459c-9ddd-2f70b6736dba","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530279848Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:26.530293272Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530472474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530499452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530513996Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.53066715Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"task":"crutest"} {"msg":"The guest has requested activation with gen:124","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530683033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} {"msg":"dbe46b28-d141-459c-9ddd-2f70b6736dba active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530700235Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530715579Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"0","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530729833Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"1","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530743607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"2","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530774073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530818754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530843223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"ds_connection connected from Ok(127.0.0.1:64315)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530858967Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"0","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"ds_connection connected from Ok(127.0.0.1:33211)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.53087412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"1","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"ds_connection connected from Ok(127.0.0.1:61128)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.530955985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"io task","client":"2","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.531578365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"0","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.531606073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"1","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.531696164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"2","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.531972564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.53202859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[0]R flush_numbers[0..12]: [250, 250, 245, 221, 244, 244, 243, 242, 234, 239, 246, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532094001Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[0]R generation[0..12]: [123, 123, 121, 107, 120, 120, 120, 119, 114, 117, 122, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532113913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532130545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[1]R flush_numbers[0..12]: [250, 250, 245, 221, 244, 244, 243, 242, 234, 239, 246, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532146569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[1]R generation[0..12]: [123, 123, 121, 107, 120, 120, 120, 119, 114, 117, 122, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532162042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532178335Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[2]R flush_numbers[0..12]: [250, 250, 245, 221, 244, 244, 243, 242, 234, 239, 246, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532193918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[2]R generation[0..12]: [123, 123, 121, 107, 120, 120, 120, 119, 114, 117, 122, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532209491Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532225684Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Max found gen is 124","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:53:26.532239178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Generation requested: 124 >= found:124","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.53225869Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Next flush: 251","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532272114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532285538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"":"downstairs","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532299632Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532312467Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"0","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:53:26.53232821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"1","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532345142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"client":"2","":"downstairs","session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532359726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.53237326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"dbe46b28-d141-459c-9ddd-2f70b6736dba is now active with session: c85fe171-6fdf-429d-be75-47aeec3eac15","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532388004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532400758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696,"session_id":"c85fe171-6fdf-429d-be75-47aeec3eac15"} {"msg":"The guest has finished waiting for activation with:124","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532414113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} {"msg":"Activated sub_volume dbe46b28-d141-459c-9ddd-2f70b6736dba","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:26.532425708Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 175 len 40960 data: 9 8 7 5 8 9 11 13 13 12 02/30 Write block 560 len 20480 data: 11 11 11 10 9 03/30 Read block 39 len 12288 04/30 Write block 132 len 8192 data: 11 11 05/30 Write block 572 len 16384 data: 10 7 8 7 06/30 Write block 419 len 32768 data: 11 9 8 10 10 10 11 9 07/30 Write block 502 len 24576 data: 14 13 12 11 13 14 08/30 Read block 318 len 28672 09/30 Read block 376 len 20480 10/30 Read block 294 len 40960 11/30 Write block 465 len 20480 data: 10 9 11 13 12 12/30 Read block 553 len 36864 13/30 Write block 277 len 12288 data: 7 6 6 14/30 Write block 162 len 12288 data: 15 11 8 15/30 Read block 215 len 28672 16/30 Write block 380 len 28672 data: 10 12 12 10 9 9 9 17/30 Read block 443 len 16384 18/30 Read block 98 len 20480 19/30 Read block 180 len 20480 20/30 Read block 506 len 24576 21/30 Write block 147 len 36864 data: 9 12 11 11 12 7 6 9 11 22/30 Read block 269 len 32768 23/30 Read block 566 len 12288 24/30 Read block 541 len 36864 25/30 Read block 64 len 12288 26/30 Write block 375 len 16384 data: 6 7 6 8 27/30 Read block 191 len 4096 28/30 Write block 218 len 8192 data: 5 6 29/30 Write block 200 len 20480 data: 13 12 14 16 16 30/30 Read block 384 len 32768 ---------------------------------------------------------------- Crucible gen:124 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 5 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 2 Done Done Sent false 1004 Acked Write 4 Done Done Sent false 1005 Acked Write 8 Done Done Sent false 1006 Acked Write 6 Done Done Sent false 1007 Acked Read 7 Done Done Sent false 1008 Acked Read 5 Done Done Sent false 1009 Acked Read 10 Done Done Sent false 1010 Acked Write 5 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Write 3 Done Done Sent false 1013 Acked Write 3 Done Done Sent false 1014 Acked Read 7 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 4 Done Done Sent false 1017 Acked Read 5 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 9 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Read 3 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Read 3 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 1 Done Done Sent false 1027 Acked Write 2 Done Done Sent false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 8 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 27 29 Done 30 28 3 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None None None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:26.544387144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1696} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 250 250 250 F F F 1 020-039 123 123 123 250 250 250 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 120 120 120 243 243 243 T T F <--- 7 140-159 119 119 119 242 242 242 T T F <--- 8 160-179 114 114 114 234 234 234 T T T 9 180-199 117 117 117 239 239 239 T T T 10 200-219 122 122 122 246 246 246 T T F <--- 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 120 120 120 244 244 244 T T F <--- 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 122 122 122 247 247 247 F F F 17 340-359 122 122 122 246 246 246 F F F 18 360-379 119 119 119 242 242 242 T T F <--- 19 380-399 122 122 122 249 249 249 T T F <--- 20 400-419 115 115 115 236 236 236 T T F <--- 21 420-439 121 121 121 245 245 245 T T F <--- 22 440-459 115 115 115 236 236 236 F F F 23 460-479 119 119 119 242 242 242 T T F <--- 24 480-499 122 122 122 247 247 247 F F F 25 500-519 120 120 120 243 243 243 T T F <--- 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 121 121 121 245 245 245 T T T 29 580-599 121 121 121 245 245 245 F F F Max gen: 123, Max flush: 250 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 62 Verifying data now No change after verify Loop: 62 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 123 123 123 250 250 250 F F F 1 020-039 123 123 123 250 250 250 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 120 120 120 244 244 244 F F F 5 100-119 120 120 120 244 244 244 F F F 6 120-139 124 124 124 251 251 251 F F F 7 140-159 124 124 124 251 251 251 F F F 8 160-179 124 124 124 251 251 251 F F F 9 180-199 124 124 124 251 251 251 F F F 10 200-219 124 124 124 251 251 251 F F F 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 124 124 124 251 251 251 F F F 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 122 122 122 247 247 247 F F F 17 340-359 122 122 122 246 246 246 F F F 18 360-379 124 124 124 251 251 251 F F F 19 380-399 124 124 124 251 251 251 F F F 20 400-419 124 124 124 251 251 251 F F F 21 420-439 124 124 124 251 251 251 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 122 122 122 247 247 247 F F F 25 500-519 124 124 124 251 251 251 F F F 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 124 124 124 251 251 251 F F F 29 580-599 121 121 121 245 245 245 F F F Max gen: 124, Max flush: 251 Begin loop 63 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.873581402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.885366255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.885686566Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.885726828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.885739073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.885757065Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} {"msg":"Crucible 6580ca27-7c31-4afe-b824-faf7faef89ae has session id: cea58a1e-df6c-47dd-905a-229c94975ce1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.8861389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Upstairs opts: Upstairs UUID: 6580ca27-7c31-4afe-b824-faf7faef89ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886171576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Crucible stats registered with UUID: 6580ca27-7c31-4afe-b824-faf7faef89ae","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886186509Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:30.886199244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886378306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.88641504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886430583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886526132Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"task":"crutest"} {"msg":"The guest has requested activation with gen:126","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886543165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} {"msg":"6580ca27-7c31-4afe-b824-faf7faef89ae active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886560537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886573162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"0","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886586986Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"1","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.88660053Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"2","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886614234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886628058Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886667221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"ds_connection connected from Ok(127.0.0.1:62108)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886683614Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"2","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"ds_connection connected from Ok(127.0.0.1:34797)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886700696Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"1","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"ds_connection connected from Ok(127.0.0.1:47294)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.886844084Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"io task","client":"0","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887395345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"2","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887432189Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"0","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887449142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"1","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887770643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887809306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[0]R flush_numbers[0..12]: [250, 250, 245, 221, 244, 244, 251, 251, 251, 251, 251, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887871519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[0]R generation[0..12]: [123, 123, 121, 107, 120, 120, 124, 124, 124, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887888412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887904585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[1]R flush_numbers[0..12]: [250, 250, 245, 221, 244, 244, 251, 251, 251, 251, 251, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887920318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[1]R generation[0..12]: [123, 123, 121, 107, 120, 120, 124, 124, 124, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887935661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887952064Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[2]R flush_numbers[0..12]: [250, 250, 245, 221, 244, 244, 251, 251, 251, 251, 251, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887968047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[2]R generation[0..12]: [123, 123, 121, 107, 120, 120, 124, 124, 124, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.887986629Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888002942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Max found gen is 125","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888016396Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Generation requested: 126 >= found:125","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.88803063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Next flush: 252","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888043804Z","hostname":"Wait for a query_work_queue command to finish before sending IO w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888062316Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"":"downstairs","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.8880762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888088725Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"0","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888102609Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"1","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888116293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"client":"2","":"downstairs","session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888130297Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:53:30.888143611Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"6580ca27-7c31-4afe-b824-faf7faef89ae is now active with session: cea58a1e-df6c-47dd-905a-229c94975ce1","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888161004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888173748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704,"session_id":"cea58a1e-df6c-47dd-905a-229c94975ce1"} {"msg":"The guest has finished waiting for activation with:126","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888187242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} {"msg":"Activated sub_volume 6580ca27-7c31-4afe-b824-faf7faef89ae","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:30.888200356Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 13 len 28672 data: 9 11 11 12 11 9 8 02/30 Read block 10 len 36864 03/30 Read block 503 len 20480 04/30 Write block 429 len 8192 data: 9 8 05/30 Read block 89 len 16384 06/30 Write block 26 len 36864 data: 10 9 11 14 13 13 12 13 13 07/30 Write block 111 len 28672 data: 6 6 6 7 7 9 7 08/30 Read block 82 len 16384 09/30 Read block 511 len 8192 10/30 Read block 312 len 4096 11/30 Flush 12/30 Write block 108 len 16384 data: 8 7 8 7 13/30 Read block 409 len 36864 14/30 Read block 406 len 20480 15/30 Write block 91 len 36864 data: 6 11 12 12 13 12 12 13 12 16/30 Write block 116 len 12288 data: 10 8 8 17/30 Flush 18/30 Write block 140 len 28672 data: 9 10 10 8 9 9 8 19/30 Write block 168 len 16384 data: 13 14 14 14 20/30 Read block 315 len 16384 21/30 Read block 12 len 12288 22/30 Write block 505 len 24576 data: 12 14 15 14 13 14 23/30 Read block 442 len 12288 24/30 Read block 173 len 36864 25/30 Write block 374 len 20480 data: 6 7 8 7 9 26/30 Read block 17 len 28672 27/30 Read block 219 len 20480 28/30 Write block 343 len 4096 data: 12 29/30 Write block 121 len 24576 data: 11 14 14 14 14 12 30/30 Read block 435 len 4096 ---------------------------------------------------------------- Crucible gen:126 GIO:true work queues: Upstairs:0 downstairs:19 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1011 Acked Write 4 Done Done Done false 1012 Acked Read 9 Done Done Done false 1013 Acked Read 5 Done Done Done false 1014 Acked Write 9 Done Sent Done false 1015 Acked Write 3 Done Sent Done false 1016 Acked Flush 0 Done Sent Done false 1017 Acked Write 7 Done Sent Done false 1018 Acked Write 4 Done Sent Done false 1019 Acked Read 4 Done Sent Done false 1020 Acked Read 3 Done Sent Done false 1021 Acked Write 6 Done Sent Done false 1022 Acked Read 3 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Write 5 Done Sent Done false 1025 Acked Read 7 Done Sent Done false 1026 Acked Read 5 Done Sent Done false 1027 Acked Write 1 Done Sent Done false 1028 Acked Write 6 Done Sent Done false 1029 Acked Read 1 Done Sent Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 16 0 16 Done 19 3 19 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1016)) Some(JobId(1010)) Some(JobId(1016)) Downstairs last five completed: 1010 1009 1008 1007 1006 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:30.900607539Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1704} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 252 252 252 F F F 1 020-039 126 126 126 252 252 252 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 126 120 126 253 244 253 F F F <--- 5 100-119 126 126 126 253 252 253 F T F <--- 6 120-139 124 124 124 251 251 251 T F T <--- 7 140-159 124 124 124 251 251 251 T F T <--- 8 160-179 124 124 124 251 251 251 T F T <--- 9 180-199 124 124 124 251 251 251 F F F 10 200-219 124 124 124 251 251 251 F F F 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 124 124 124 251 251 251 F F F 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 122 122 122 247 247 247 F F F 17 340-359 122 122 122 246 246 246 T F T <--- 18 360-379 124 124 124 251 251 251 T F T <--- 19 380-399 124 124 124 251 251 251 F F F 20 400-419 124 124 124 251 251 251 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 122 122 122 247 247 247 F F F 25 500-519 124 124 124 251 251 251 T F T <--- 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 124 124 124 251 251 251 F F F 29 580-599 121 121 121 245 245 245 F F F Max gen: 126, Max flush: 253 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 63 Verifying data now No change after verify Loop: 63 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 126 126 126 252 252 252 F F F 1 020-039 126 126 126 252 252 252 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F F F 4 080-099 126 126 126 253 253 253 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 127 127 127 254 254 254 F F F 8 160-179 127 127 127 254 254 254 F F F 9 180-199 124 124 124 251 251 251 F F F 10 200-219 124 124 124 251 251 251 F F F 11 220-239 120 120 120 243 243 243 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 124 124 124 251 251 251 F F F 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 122 122 122 247 247 247 F F F 17 340-359 127 127 127 254 254 254 F F F 18 360-379 127 127 127 254 254 254 F F F 19 380-399 124 124 124 251 251 251 F F F 20 400-419 124 124 124 251 251 251 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 122 122 122 247 247 247 F F F 25 500-519 127 127 127 254 254 254 F F F 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 124 124 124 251 251 251 F F F 29 580-599 121 121 121 245 245 245 F F F Max gen: 127, Max flush: 254 Begin loop 64 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.215046984Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:35.225706404Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.225755253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.236305291Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.236636127Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.236675181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.236687395Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.236705208Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} {"msg":"Crucible 8208d651-402b-46ef-a9ff-63822dafc8bf has session id: d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237052707Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Upstairs opts: Upstairs UUID: 8208d651-402b-46ef-a9ff-63822dafc8bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237088312Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Crucible stats registered with UUID: 8208d651-402b-46ef-a9ff-63822dafc8bf","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237104345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:35.237117979Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237288165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237314094Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237329087Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237455232Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"task":"crutest"} {"msg":"The guest has requested activation with gen:128","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237471925Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} {"msg":"8208d651-402b-46ef-a9ff-63822dafc8bf active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237487688Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237500603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"0","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237514497Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"1","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237528591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"2","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237542705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237556649Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237570842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:62130)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237613344Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"0","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:37078)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237628587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"1","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"ds_connection connected from Ok(127.0.0.1:41575)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.237643591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"io task","client":"2","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238264382Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"2","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238349525Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"1","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238421554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"0","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238609562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238699853Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[0]R flush_numbers[0..12]: [252, 252, 245, 221, 253, 253, 254, 254, 254, 251, 251, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238817162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[0]R generation[0..12]: [126, 126, 121, 107, 126, 126, 127, 127, 127, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238834425Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238851038Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[1]R flush_numbers[0..12]: [252, 252, 245, 221, 253, 253, 254, 254, 254, 251, 251, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238866961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[1]R generation[0..12]: [126, 126, 121, 107, 126, 126, 127, 127, 127, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238882444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238898587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[2]R flush_numbers[0..12]: [252, 252, 245, 221, 253, 253, 254, 254, 254, 251, 251, 243]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.23891416Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[2]R generation[0..12]: [126, 126, 121, 107, 126, 126, 127, 127, 127, 124, 124, 120]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238930213Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238946496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Max found gen is 128","vWait for a query_work_queue command to finish before sending IO ":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238965728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Generation requested: 128 >= found:128","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238980132Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Next flush: 255","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.238993436Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.23900702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"":"downstairs","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239021034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239035757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"0","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:53:35.239049492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"1","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239066454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"client":"2","":"downstairs","session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239080888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239094222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"8208d651-402b-46ef-a9ff-63822dafc8bf is now active with session: d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239109306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.2391219Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712,"session_id":"d4e81e8e-0b8f-4c48-849e-9cfc6c9d8ffd"} {"msg":"The guest has finished waiting for activation with:128","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239135734Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} {"msg":"Activated sub_volume 8208d651-402b-46ef-a9ff-63822dafc8bf","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:35.239147069Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 272 len 20480 data: 10 10 9 8 7 02/30 Write block 325 len 4096 data: 11 03/30 Write block 18 len 32768 data: 10 9 8 11 11 13 12 10 04/30 Flush 05/30 Write block 586 len 8192 data: 10 10 06/30 Read block 192 len 40960 07/30 Read block 141 len 16384 08/30 Write block 579 len 4096 data: 9 09/30 Read block 229 len 40960 10/30 Read block 95 len 40960 11/30 Write block 373 len 32768 data: 6 7 8 9 8 10 8 11 12/30 Read block 457 len 36864 13/30 Write block 202 len 28672 data: 15 17 17 16 11 10 8 14/30 Write block 232 len 28672 data: 8 10 10 10 11 11 13 15/30 Read block 295 len 12288 16/30 Write block 188 len 8192 data: 9 9 17/30 Write block 87 len 16384 data: 9 8 6 6 18/30 Write block 490 len 12288 data: 12 11 11 19/30 Read block 523 len 8192 20/30 Read block 328 len 36864 21/30 Read block 274 len 28672 22/30 Write block 262 len 28672 data: 15 11 11 12 13 11 10 23/30 Read block 238 len 24576 24/30 Write block 72 len 16384 data: 6 6 12 12 25/30 Read block 319 len 28672 26/30 Read block 166 len 36864 27/30 Read block 300 len 24576 28/30 Write block 28 len 12288 data: 12 15 14 29/30 Read block 197 len 24576 30/30 Write block 272 len 12288 data: 11 11 10 ---------------------------------------------------------------- Crucible gen:128 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 2 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 4 Done Done Done false 1007 Acked Write 1 Done Done Done false 1008 Acked Read 10 Done Done Done false 1009 Acked Read 10 Done Done Done false 1010 Acked Write 8 Done Done Done false 1011 Acked Read 9 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Write 7 Sent Done Done false 1014 Acked Read 3 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Write 4 Sent Done Done false 1017 Acked Write 3 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Read 9 Sent Done Done false 1020 Acked Read 7 Sent Done Done false 1021 Acked Write 7 Sent Done Done false 1022 Acked Read 6 Sent Done Done false 1023 Acked Write 4 Sent Done Done false 1024 Acked Read 7 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 3 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 19 1 1 21 Done 7 25 25 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1003)) Some(JobId(1003)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:35.250878745Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1712} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 255 255 255 F F F 1 020-039 128 128 128 255 255 255 F T T <--- 2 040-059 121 121 121 245 245 245 F F F 3 060-079 107 107 107 221 221 221 F T T <--- 4 080-099 126 126 126 253 253 253 F T T <--- 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 127 127 127 254 254 254 F F F 8 160-179 127 127 127 254 254 254 F F F 9 180-199 124 124 124 251 251 251 F T T <--- 10 200-219 124 124 124 251 251 251 F T T <--- 11 220-239 120 120 120 243 243 243 F T T <--- 12 240-259 119 119 119 242 242 242 F F F 13 260-279 128 128 128 255 255 255 F T T <--- 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 128 128 128 255 255 255 F F F 17 340-359 127 127 127 254 254 254 F F F 18 360-379 127 127 127 254 254 254 T T T 19 380-399 124 124 124 251 251 251 T T T 20 400-419 124 124 124 251 251 251 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 122 122 122 247 247 247 F T T <--- 25 500-519 127 127 127 254 254 254 F F F 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 124 124 124 251 251 251 T T T 29 580-599 121 121 121 245 245 245 T T T Max gen: 128, Max flush: 255 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 64 Verifying data now No change after verify Loop: 64 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 255 255 255 F F F 1 020-039 129 129 129 256 256 256 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 129 129 129 256 256 256 F F F 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 127 127 127 254 254 254 F F F 8 160-179 127 127 127 254 254 254 F F F 9 180-199 129 129 129 256 256 256 F F F 10 200-219 129 129 129 256 256 256 F F F 11 220-239 129 129 129 256 256 256 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 129 129 129 256 256 256 F F F 14 280-299 122 122 122 249 249 249 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 128 128 128 255 255 255 F F F 17 340-359 127 127 127 254 254 254 F F F 18 360-379 129 129 129 256 256 256 F F F 19 380-399 129 129 129 256 256 256 F F F 20 400-419 124 124 124 251 251 251 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 129 129 129 256 256 256 F F F 25 500-519 127 127 127 254 254 254 F F F 26 520-539 122 122 122 246 246 246 F F F 27 540-559 114 114 114 234 234 234 F F F 28 560-579 129 129 129 256 256 256 F F F 29 580-599 129 129 129 256 256 256 F F F Max gen: 129, Max flush: 256 Begin loop 65 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.585064892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:39.595734148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.59577569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606000658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606287484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606326557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606338642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606356334Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} {"msg":"Crucible 53f59f84-c4bc-49b2-ba61-4f81d4f9b9d9 has session id: bf37f53b-c56b-4756-8891-f7dfb42fa9fc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.60668845Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Upstairs opts: Upstairs UUID: 53f59f84-c4bc-49b2-ba61-4f81d4f9b9d9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606723305Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Crucible stats registered with UUID: 53f59f84-c4bc-49b2-ba61-4f81d4f9b9d9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606739078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:39.606752742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606927766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606964151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.606980064Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607087047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"task":"crutest"} {"msg":"The guest has requested activation with gen:130","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.60710398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} {"msg":"53f59f84-c4bc-49b2-ba61-4f81d4f9b9d9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607121503Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607134067Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"0","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607148111Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"1","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607161905Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"2","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607175499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607189423Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607205746Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41046)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607230975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"1","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:45422)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607246029Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"2","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"ds_connection connected from Ok(127.0.0.1:63054)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607414875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"io task","client":"0","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607890949Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"1","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.607919677Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"2","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608079687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"0","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608283839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608329619Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[0]R flush_numbers[0..12]: [255, 256, 245, 256, 256, 253, 254, 254, 254, 256, 256, 256]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608456214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[0]R generation[0..12]: [128, 129, 121, 129, 129, 126, 127, 127, 127, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608473207Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.60848968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[1]R flush_numbers[0..12]: [255, 256, 245, 256, 256, 253, 254, 254, 254, 256, 256, 256]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608505613Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[1]R generation[0..12]: [128, 129, 121, 129, 129, 126, 127, 127, 127, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608521206Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608540528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[2]R flush_numbers[0..12]: [255, 256, 245, 256, 256, 253, 254, 254, 254, 256, 256, 256]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608556631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[2]R generation[0..12]: [128, 129, 121, 129, 129, 126, 127, 127, 127, 129, 129, 129]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608572324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608588497Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Max found gen is 130","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608601851Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} Wait for a query_work_queue command to finish before sending IO {"msg":"Generation requested: 130 >= found:130","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608621412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Next flush: 257","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608634697Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608648241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"":"downstairs","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608662025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608674839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"0","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608688773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"1","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608702387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"client":"2","":"downstairs","session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible",Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "level":30,"time":"2026-03-12T22:53:39.60871906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608732704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"53f59f84-c4bc-49b2-ba61-4f81d4f9b9d9 is now active with session: bf37f53b-c56b-4756-8891-f7dfb42fa9fc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608747428Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608761652Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720,"session_id":"bf37f53b-c56b-4756-8891-f7dfb42fa9fc"} {"msg":"The guest has finished waiting for activation with:130","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608775066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} {"msg":"Activated sub_volume 53f59f84-c4bc-49b2-ba61-4f81d4f9b9d9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:39.608786391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 577 len 20480 data: 5 8 10 9 9 03/30 Write block 490 len 24576 data: 13 12 12 10 12 14 04/30 Write block 504 len 32768 data: 13 13 15 16 15 14 15 15 05/30 Read block 188 len 36864 06/30 Write block 219 len 4096 data: 7 07/30 Read block 284 len 4096 08/30 Read block 6 len 32768 09/30 Write block 325 len 32768 data: 12 12 13 14 10 11 10 11 10/30 Write block 285 len 32768 data: 15 18 19 16 15 13 15 13 11/30 Read block 250 len 40960 12/30 Flush 13/30 Write block 534 len 12288 data: 8 6 6 14/30 Write block 587 len 24576 data: 11 10 12 13 12 11 15/30 Read block 548 len 32768 16/30 Write block 266 len 16384 data: 14 12 11 10 17/30 Read block 490 len 32768 18/30 Write block 589 len 40960 data: 13 14 13 12 11 10 9 8 7 5 19/30 Read block 61 len 4096 20/30 Write block 223 len 4096 data: 12 21/30 Flush 22/30 Write block 540 len 8192 data: 6 7 23/30 Flush 24/30 Write block 29 len 12288 data: 16 15 14 25/30 Flush 26/30 Write block 201 len 4096 data: 13 27/30 Write block 282 len 16384 data: 7 6 10 16 28/30 Write block 407 len 20480 data: 10 10 12 11 12 29/30 Read block 558 len 12288 30/30 Write block 203 len 12288 data: 18 18 17 ---------------------------------------------------------------- Crucible gen:130 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 5 Done Done Done false 1002 Acked Write 6 Sent Done Done false 1003 Acked Write 8 Sent Done Done false 1004 Acked Read 9 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 8 Sent Done Done false 1008 Acked Write 8 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 3 Sent Done Done false 1013 Acked Write 6 Sent Done Done false 1014 Acked Read 8 Sent Done Done false 1015 Acked Write 4 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Read 1 Sent Done Done false 1019 Acked Write 1 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 2 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 3 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 1 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 28 1 1 30 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:39.619773599Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1720} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 255 255 255 F F F 1 020-039 129 130 130 256 261 261 F F F <--- 2 040-059 121 121 121 245 245 245 F F F 3 060-079 129 129 129 256 256 256 F F F 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 127 127 127 254 254 254 F F F 8 160-179 127 127 127 254 254 254 F F F 9 180-199 129 129 129 256 256 256 F F F 10 200-219 129 130 130 256 258 258 F T T <--- 11 220-239 129 130 130 256 259 259 F F F <--- 12 240-259 119 119 119 242 242 242 F F F 13 260-279 129 130 130 256 259 259 F F F <--- 14 280-299 122 130 130 249 258 258 F T T <--- 15 300-319 122 122 122 249 249 249 F F F 16 320-339 128 130 130 255 258 258 F F F <--- 17 340-359 127 127 127 254 254 254 F F F 18 360-379 129 129 129 256 256 256 F F F 19 380-399 129 129 129 256 256 256 F F F 20 400-419 124 124 124 251 251 251 F T T <--- 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 129 130 130 256 258 258 F F F <--- 25 500-519 127 130 130 254 258 258 F F F <--- 26 520-539 122 130 130 246 259 259 F F F <--- 27 540-559 114 130 130 234 260 260 F F F <--- 28 560-579 129 130 130 256 258 258 T F F <--- 29 580-599 129 130 130 256 259 259 T F F <--- Max gen: 130, Max flush: 261 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 65 Verifying data now No change after verify Loop: 65 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 255 255 255 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 129 129 129 256 256 256 F F F 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 127 127 127 254 254 254 F F F 8 160-179 127 127 127 254 254 254 F F F 9 180-199 129 129 129 256 256 256 F F F 10 200-219 131 131 131 262 262 262 F F F 11 220-239 130 130 130 259 259 259 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 130 130 130 259 259 259 F F F 14 280-299 131 131 131 262 262 262 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 130 130 130 258 258 258 F F F 17 340-359 127 127 127 254 254 254 F F F 18 360-379 129 129 129 256 256 256 F F F 19 380-399 129 129 129 256 256 256 F F F 20 400-419 131 131 131 262 262 262 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 124 124 124 251 251 251 F F F 24 480-499 130 130 130 258 258 258 F F F 25 500-519 130 130 130 258 258 258 F F F 26 520-539 130 130 130 259 259 259 F F F 27 540-559 130 130 130 260 260 260 F F F 28 560-579 130 130 130 258 258 258 F F F 29 580-599 130 130 130 259 259 259 F F F Max gen: 131, Max flush: 262 Begin loop 66 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.951553348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.963465655Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.963790325Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.963829258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.963841192Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.963859295Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} {"msg":"Crucible 2d872028-8a9b-409f-8e5e-c6f7b6c11fc7 has session id: 10e39fef-bf69-4627-81e4-bc28655462b9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964210932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Upstairs opts: Upstairs UUID: 2d872028-8a9b-409f-8e5e-c6f7b6c11fc7, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964283971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Crucible stats registered with UUID: 2d872028-8a9b-409f-8e5e-c6f7b6c11fc7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964300154Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:43.964313418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964490081Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964525036Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964542758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.96469995Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"task":"crutest"} {"msg":"The guest has requested activation with gen:132","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964713994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} {"msg":"2d872028-8a9b-409f-8e5e-c6f7b6c11fc7 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964725849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964738694Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"0","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964753227Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"1","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964767201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"2","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964781075Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964795489Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.96481515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:55568)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964876104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"2","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:47475)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964946703Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"0","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"ds_connection connected from Ok(127.0.0.1:57523)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.964996332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"io task","client":"1","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.965682704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"1","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.965710512Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"2","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.965730394Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"0","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966092647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966138637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[0]R flush_numbers[0..12]: [255, 261, 245, 256, 256, 253, 254, 254, 254, 256, 262, 259]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966185996Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[0]R generation[0..12]: [128, 130, 121, 129, 129, 126, 127, 127, 127, 129, 131, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966203169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966219822Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[1]R flush_numbers[0..12]: [255, 261, 245, 256, 256, 253, 254, 254, 254, 256, 262, 259]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966235495Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[1]R generation[0..12]: [128, 130, 121, 129, 129, 126, 127, 127, 127, 129, 131, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966251088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966267441Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[2]R flush_numbers[0..12]: [255, 261, 245, 256, 256, 253, 254, 254, 254, 256, 262, 259]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966283144Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[2]R generation[0..12]: [128, 130, 121, 129, 129, 126, 127, 127, 127, 129, 131, 130]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966299057Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.96631561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 132","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966335052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Generation requested: 132 >= found:132","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966349635Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Next flush: 263","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966365179Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966378693Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"":"downstairs","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966393326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966406071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"0","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:53:43.966420954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"1","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966437197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"client":"2","":"downstairs","session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966451631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966465155Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"2d872028-8a9b-409f-8e5e-c6f7b6c11fc7 is now active with session: 10e39fef-bf69-4627-81e4-bc28655462b9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966480249Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966492673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728,"session_id":"10e39fef-bf69-4627-81e4-bc28655462b9"} {"msg":"The guest has finished waiting for activation with:132","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966506148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} {"msg":"Activated sub_volume 2d872028-8a9b-409f-8e5e-c6f7b6c11fc7","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:43.966517762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 392 len 28672 data: 13 12 8 5 5 5 6 02/30 Write block 351 len 32768 data: 13 13 12 12 12 14 15 16 03/30 Read block 369 len 12288 04/30 Write block 223 len 28672 data: 13 11 6 5 6 8 8 05/30 Write block 469 len 20480 data: 13 11 11 10 9 06/30 Read block 372 len 12288 07/30 Flush 08/30 Write block 202 len 20480 data: 16 19 19 18 12 09/30 Read block 270 len 12288 10/30 Read block 226 len 16384 11/30 Read block 412 len 28672 12/30 Read block 117 len 20480 13/30 Flush 14/30 Write block 565 len 16384 data: 10 8 8 10 15/30 Read block 494 len 24576 16/30 Read block 368 len 8192 17/30 Write block 158 len 12288 data: 10 11 15 18/30 Write block 566 len 4096 data: 9 19/30 Read block 374 len 24576 20/30 Read block 504 len 20480 21/30 Read block 334 len 4096 22/30 Write block 73 len 8192 data: 7 13 23/30 Read block 19 len 16384 24/30 Write block 176 len 24576 data: 9 8 6 9 10 12 25/30 Read block 30 len 24576 26/30 Write block 493 len 32768 data: 11 13 15 16 15 14 17 12 27/30 Write block 277 len 40960 data: 8 7 7 6 4 8 7 11 17 19 28/30 Read block 439 len 8192 29/30 Read block 117 len 12288 30/30 Write block 196 len 28672 data: 13 16 16 15 14 14 17 ---------------------------------------------------------------- Crucible gen:132 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Read 3 Done Done Done false 1003 Acked Write 7 Done Done Sent false 1004 Acked Write 5 Done Done Sent false 1005 Acked Read 3 Done Done Sent false 1006 Acked Flush 0 Done Done Sent false 1007 Acked Write 5 Done Done Sent false 1008 Acked Read 3 Done Done Sent false 1009 Acked Read 4 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 4 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 2 Done Done Sent false 1016 Acked Write 3 Done Done Sent false 1017 Acked Write 1 Done Done Sent false 1018 Acked Read 6 Done Done Sent false 1019 Acked Read 5 Done Done Sent false 1020 Acked Read 1 Done Done Sent false 1021 Acked Write 2 Done Done Sent false 1022 Acked Read 4 Done Done Sent false 1023 Acked Write 6 Done Done Sent false 1024 Acked Read 6 Done Done Sent false 1025 Acked Write 8 Done Done Sent false 1026 Acked Write 10 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 27 29 Done 29 29 3 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:43.97829024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1728} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 255 255 255 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 129 129 129 256 256 256 T T F <--- 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 127 127 127 254 254 254 T T F <--- 8 160-179 127 127 127 254 254 254 T T F <--- 9 180-199 129 129 129 256 256 256 T T F <--- 10 200-219 132 132 131 264 264 262 T T F <--- 11 220-239 132 132 130 263 263 259 F F F <--- 12 240-259 119 119 119 242 242 242 F F F 13 260-279 130 130 130 259 259 259 T T F <--- 14 280-299 131 131 131 262 262 262 T T F <--- 15 300-319 122 122 122 249 249 249 F F F 16 320-339 130 130 130 258 258 258 F F F 17 340-359 132 132 127 263 263 254 F F T <--- 18 360-379 129 129 129 256 256 256 F F F 19 380-399 132 132 129 263 263 256 F F T <--- 20 400-419 131 131 131 262 262 262 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 132 132 124 263 263 251 F F F <--- 24 480-499 130 130 130 258 258 258 T T F <--- 25 500-519 130 130 130 258 258 258 T T F <--- 26 520-539 130 130 130 259 259 259 F F F 27 540-559 130 130 130 260 260 260 F F F 28 560-579 130 130 130 258 258 258 T T F <--- 29 580-599 130 130 130 259 259 259 F F F Max gen: 132, Max flush: 264 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 66 Verifying data now No change after verify Loop: 66 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 128 128 255 255 255 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 133 133 133 265 265 265 F F F 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 133 133 133 265 265 265 F F F 8 160-179 133 133 133 265 265 265 F F F 9 180-199 133 133 133 265 265 265 F F F 10 200-219 133 133 133 265 265 265 F F F 11 220-239 132 132 132 263 263 263 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 133 133 133 265 265 265 F F F 14 280-299 133 133 133 265 265 265 F F F 15 300-319 122 122 122 249 249 249 F F F 16 320-339 130 130 130 258 258 258 F F F 17 340-359 132 132 132 263 263 263 F F F 18 360-379 129 129 129 256 256 256 F F F 19 380-399 132 132 132 263 263 263 F F F 20 400-419 131 131 131 262 262 262 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 115 115 236 236 236 F F F 23 460-479 132 132 132 263 263 263 F F F 24 480-499 133 133 133 265 265 265 F F F 25 500-519 133 133 133 265 265 265 F F F 26 520-539 130 130 130 259 259 259 F F F 27 540-559 130 130 130 260 260 260 F F F 28 560-579 133 133 133 265 265 265 F F F 29 580-599 130 130 130 259 259 259 F F F Max gen: 133, Max flush: 265 Begin loop 67 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.310246443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:48.320924835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.320979821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331199872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331496644Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331537596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331553069Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331571121Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} {"msg":"Crucible 9a6f74a1-457b-4c89-8d56-3aef1b245d1e has session id: 2079df87-a2eb-49a8-a721-6856c111b627","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331929596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Upstairs opts: Upstairs UUID: 9a6f74a1-457b-4c89-8d56-3aef1b245d1e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331967829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Crucible stats registered with UUID: 9a6f74a1-457b-4c89-8d56-3aef1b245d1e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.331983582Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:48.331996937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332210324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332247478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332262961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332363008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"task":"crutest"} {"msg":"The guest has requested activation with gen:134","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332380081Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} {"msg":"9a6f74a1-457b-4c89-8d56-3aef1b245d1e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332398003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332410487Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"0","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.33242694Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"1","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332441154Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"2","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332454788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332468902Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332483136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"ds_connection connected from Ok(127.0.0.1:58264)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332498729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"2","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"ds_connection connected from Ok(127.0.0.1:64201)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332513333Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"1","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"ds_connection connected from Ok(127.0.0.1:53451)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.332528047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"io task","client":"0","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333322922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"2","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333359786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"0","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333377099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"1","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333711564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333727807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[0]R flush_numbers[0..12]: [255, 261, 245, 265, 256, 253, 254, 265, 265, 265, 265, 263]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333752446Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[0]R generation[0..12]: [128, 130, 121, 133, 129, 126, 127, 133, 133, 133, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333768459Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.33378843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[1]R flush_numbers[0..12]: [255, 261, 245, 265, 256, 253, 254, 265, 265, 265, 265, 263]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333804323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[1]R generation[0..12]: [128, 130, 121, 133, 129, 126, 127, 133, 133, 133, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333820047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333836459Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[2]R flush_numbers[0..12]: [255, 261, 245, 265, 256, 253, 254, 265, 265, 265, 265, 263]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333852303Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[2]R generation[0..12]: [128, 130, 121, 133, 129, 126, 127, 133, 133, 133, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333868326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333884329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Max found gen is 134","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333897803Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Generation requested: 134 >= found:134","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:53:48.333911777Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Next flush: 266","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333929469Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333942913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"":"downstairs","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333956907Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333969422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"0","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.333983486Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"1","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.33399769Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"client":"2","":"downstairs","session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:53:48.334013733Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.334030116Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"9a6f74a1-457b-4c89-8d56-3aef1b245d1e is now active with session: 2079df87-a2eb-49a8-a721-6856c111b627","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.334045379Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.334058223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736,"session_id":"2079df87-a2eb-49a8-a721-6856c111b627"} {"msg":"The guest has finished waiting for activation with:134","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.334071538Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} {"msg":"Activated sub_volume 9a6f74a1-457b-4c89-8d56-3aef1b245d1e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:48.334083163Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 411 len 4096 data: 13 02/30 Read block 570 len 16384 03/30 Read block 77 len 36864 04/30 Write block 9 len 24576 data: 12 13 12 11 10 12 05/30 Read block 95 len 20480 06/30 Flush 07/30 Write block 537 len 36864 data: 5 5 6 7 8 8 7 7 9 08/30 Flush 09/30 Write block 448 len 4096 data: 6 10/30 Write block 299 len 24576 data: 12 11 12 8 8 7 11/30 Read block 36 len 16384 12/30 Read block 441 len 40960 13/30 Read block 182 len 28672 14/30 Write block 152 len 36864 data: 8 7 10 12 11 9 11 12 16 15/30 Write block 186 len 24576 data: 13 11 10 10 11 13 16/30 Write block 159 len 8192 data: 13 17 17/30 Read block 343 len 20480 18/30 Write block 270 len 40960 data: 9 10 12 12 11 9 8 9 8 8 19/30 Flush 20/30 Write block 370 len 20480 data: 10 8 8 7 8 21/30 Write block 510 len 20480 data: 16 16 15 11 11 22/30 Read block 378 len 16384 23/30 Read block 323 len 4096 24/30 Read block 7 len 36864 25/30 Flush 26/30 Write block 569 len 36864 data: 12 10 7 11 8 9 8 6 6 27/30 Read block 208 len 40960 28/30 Write block 464 len 40960 data: 13 11 10 12 14 14 12 12 11 10 29/30 Read block 178 len 24576 30/30 Read block 590 len 8192 ---------------------------------------------------------------- Crucible gen:134 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Sent Done Done false 1001 Acked Read 4 Sent Done Done false 1002 Acked Read 9 Sent Done Done false 1003 Acked Write 6 Sent Done Done false 1004 Acked Read 5 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 9 Sent Done Done false 1007 Acked Flush 0 Sent Done Done false 1008 Acked Write 1 Sent Done Done false 1009 Acked Write 6 Sent Done Done false 1010 Acked Read 4 Sent Done Done false 1011 Acked Read 10 Sent Done Done false 1012 Acked Read 7 Sent Done Done false 1013 Acked Write 9 Sent Done Done false 1014 Acked Write 6 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Write 10 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 5 Sent Done Done false 1020 Acked Write 5 Sent Done Done false 1021 Acked Read 4 Sent Done Done false 1022 Acked Read 1 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Flush 0 Sent Done Done false 1025 Acked Write 9 Sent Done Done false 1026 Acked Read 10 Sent Done Done false 1027 Acked Write 10 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Read 2 Sent Done Done false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 0 30 Done 0 30 30 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1024)) Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:48.347827926Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1736} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 128 134 134 255 266 266 F F F <--- 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 133 133 133 265 265 265 F F F 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 133 134 134 265 268 268 F F F <--- 8 160-179 133 134 134 265 268 268 F F F <--- 9 180-199 133 134 134 265 268 268 F F F <--- 10 200-219 133 133 133 265 265 265 F F F 11 220-239 132 132 132 263 263 263 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 133 134 134 265 268 268 F F F <--- 14 280-299 133 134 134 265 268 268 F F F <--- 15 300-319 122 134 134 249 268 268 F F F <--- 16 320-339 130 130 130 258 258 258 F F F 17 340-359 132 132 132 263 263 263 F F F 18 360-379 129 134 134 256 269 269 F F F <--- 19 380-399 132 132 132 263 263 263 F F F 20 400-419 131 134 134 262 266 266 F F F <--- 21 420-439 126 126 126 252 252 252 F F F 22 440-459 115 134 134 236 268 268 F F F <--- 23 460-479 132 132 132 263 263 263 F T T <--- 24 480-499 133 133 133 265 265 265 F F F 25 500-519 133 134 134 265 269 269 F F F <--- 26 520-539 130 134 134 259 267 267 F F F <--- 27 540-559 130 134 134 260 267 267 F F F <--- 28 560-579 133 133 133 265 265 265 F T T <--- 29 580-599 130 130 130 259 259 259 F F F Max gen: 134, Max flush: 269 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 67 Verifying data now No change after verify Loop: 67 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 134 134 134 266 266 266 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 133 133 133 265 265 265 F F F 4 080-099 129 129 129 256 256 256 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 134 134 134 268 268 268 F F F 8 160-179 134 134 134 268 268 268 F F F 9 180-199 134 134 134 268 268 268 F F F 10 200-219 133 133 133 265 265 265 F F F 11 220-239 132 132 132 263 263 263 F F F 12 240-259 119 119 119 242 242 242 F F F 13 260-279 134 134 134 268 268 268 F F F 14 280-299 134 134 134 268 268 268 F F F 15 300-319 134 134 134 268 268 268 F F F 16 320-339 130 130 130 258 258 258 F F F 17 340-359 132 132 132 263 263 263 F F F 18 360-379 134 134 134 269 269 269 F F F 19 380-399 132 132 132 263 263 263 F F F 20 400-419 134 134 134 266 266 266 F F F 21 420-439 126 126 126 252 252 252 F F F 22 440-459 134 134 134 268 268 268 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 133 133 133 265 265 265 F F F 25 500-519 134 134 134 269 269 269 F F F 26 520-539 134 134 134 267 267 267 F F F 27 540-559 134 134 134 267 267 267 F F F 28 560-579 135 135 135 270 270 270 F F F 29 580-599 130 130 130 259 259 259 F F F Max gen: 135, Max flush: 270 Begin loop 68 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.682946807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.694827628Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695155066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695193829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695206194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695224096Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} {"msg":"Crucible b94541ff-bb32-4bac-855e-24c204325eb3 has session id: c90e5507-7db0-4697-b77d-50e9cad3ef98","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695572695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Upstairs opts: Upstairs UUID: b94541ff-bb32-4bac-855e-24c204325eb3, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695600253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Crucible stats registered with UUID: b94541ff-bb32-4bac-855e-24c204325eb3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695617855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:52.69563108Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695821417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695857711Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.695873165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696012264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"task":"crutest"} {"msg":"The guest has requested activation with gen:136","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696029557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} {"msg":"b94541ff-bb32-4bac-855e-24c204325eb3 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696051307Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696064292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"0","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696078216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"1","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.69609203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"2","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696105864Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696120368Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696134771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"ds_connection connected from Ok(127.0.0.1:61002)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696181391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"0","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"ds_connection connected from Ok(127.0.0.1:48419)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696225652Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"1","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"ds_connection connected from Ok(127.0.0.1:37000)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696244674Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"io task","client":"2","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696915773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"1","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.696943511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"0","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697077763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"2","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697328774Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697363829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[0]R flush_numbers[0..12]: [266, 261, 245, 265, 256, 253, 254, 268, 268, 268, 265, 263]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697454569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[0]R generation[0..12]: [134, 130, 121, 133, 129, 126, 127, 134, 134, 134, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697472442Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697489304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[1]R flush_numbers[0..12]: [266, 261, 245, 265, 256, 253, 254, 268, 268, 268, 265, 263]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697505087Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[1]R generation[0..12]: [134, 130, 121, 133, 129, 126, 127, 134, 134, 134, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.69752108Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697537303Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[2]R flush_numbers[0..12]: [266, 261, 245, 265, 256, 253, 254, 268, 268, 268, 265, 263]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697553097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[2]R generation[0..12]: [134, 130, 121, 133, 129, 126, 127, 134, 134, 134, 133, 132]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.69756893Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697588791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Max found gen is 136","v":0Wait for a query_work_queue command to finish before sending IO ,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697608183Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Generation requested: 136 >= found:136","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697622626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Next flush: 271","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697635961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697649765Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"":"downstairs","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697663729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697676423Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"0","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697690427Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"1","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.69770682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"client":"2","":"downstairs","session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697721164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697734788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"b94541ff-bb32-4bac-855e-24c204325eb3 is now active with session: c90e5507-7db0-4697-b77d-50e9cad3ef98","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697749781Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697762596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744,"session_id":"c90e5507-7db0-4697-b77d-50e9cad3ef98"} {"msg":"The guest has finished waiting for activation with:136","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.69777591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} {"msg":"Activated sub_volume b94541ff-bb32-4bac-855e-24c204325eb3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:52.697787405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 336 len 40960 data: 13 12 14 11 9 10 13 13 13 12 02/30 Write block 266 len 40960 data: 15 13 12 11 10 11 13 13 12 10 03/30 Write block 421 len 20480 data: 9 11 11 11 12 04/30 Read block 88 len 28672 05/30 Read block 377 len 4096 06/30 Write block 147 len 4096 data: 10 07/30 Write block 578 len 20480 data: 9 11 10 10 9 08/30 Write block 234 len 28672 data: 11 11 12 12 14 13 11 09/30 Read block 289 len 8192 10/30 Write block 355 len 36864 data: 13 15 16 17 16 14 15 10 11 11/30 Write block 0 len 36864 data: 2 2 4 7 10 12 12 12 15 12/30 Write block 345 len 32768 data: 13 13 11 9 14 15 14 14 13/30 Flush 14/30 Write block 428 len 28672 data: 7 10 9 8 5 5 5 15/30 Write block 77 len 32768 data: 10 10 11 13 11 10 11 11 16/30 Read block 495 len 32768 17/30 Read block 541 len 28672 18/30 Write block 341 len 24576 data: 11 14 14 14 14 14 19/30 Read block 143 len 4096 20/30 Read block 169 len 28672 21/30 Write block 421 len 40960 data: 10 12 12 12 13 10 7 8 11 10 22/30 Write block 84 len 40960 data: 12 12 11 10 9 7 7 7 12 13 23/30 Read block 12 len 20480 24/30 Read block 449 len 20480 25/30 Read block 519 len 4096 26/30 Read block 561 len 20480 27/30 Read block 138 len 8192 28/30 Read block 444 len 24576 29/30 Write block 331 len 36864 data: 11 12 10 9 12 14 13 15 12 30/30 Read block 548 len 36864 ---------------------------------------------------------------- Crucible gen:136 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 1 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 5 Done Done Sent false 1007 Acked Write 7 Done Done Sent false 1008 Acked Read 2 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 7 Done Done Sent false 1014 Acked Write 8 Done Done Sent false 1015 Acked Read 8 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Read 7 Done Done Sent false 1020 Acked Write 10 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Read 1 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 26 27 Done 30 29 4 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1012)) Some(JobId(1012)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:52.710784542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1744} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 134 271 271 266 F F F <--- 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 133 133 133 265 265 265 T T F <--- 4 080-099 129 129 129 256 256 256 T T F <--- 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 136 136 134 271 271 268 F F F <--- 8 160-179 134 134 134 268 268 268 F F F 9 180-199 134 134 134 268 268 268 F F F 10 200-219 133 133 133 265 265 265 F F F 11 220-239 136 136 132 271 271 263 F F F <--- 12 240-259 136 136 119 271 271 242 F F F <--- 13 260-279 136 136 134 271 271 268 F F T <--- 14 280-299 134 134 134 268 268 268 F F F 15 300-319 134 134 134 268 268 268 F F F 16 320-339 136 136 130 271 271 258 T T T <--- 17 340-359 136 136 132 271 271 263 T T T <--- 18 360-379 136 136 134 271 271 269 F F F <--- 19 380-399 132 132 132 263 263 263 F F F 20 400-419 134 134 134 266 266 266 F F F 21 420-439 136 136 126 271 271 252 T T T <--- 22 440-459 134 134 134 268 268 268 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 133 133 133 265 265 265 F F F 25 500-519 134 134 134 269 269 269 F F F 26 520-539 134 134 134 267 267 267 F F F 27 540-559 134 134 134 267 267 267 F F F 28 560-579 136 136 135 271 271 270 F F F <--- 29 580-599 136 136 130 271 271 259 F F F <--- Max gen: 136, Max flush: 271 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 68 Verifying data now No change after verify Loop: 68 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 271 271 271 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 121 121 121 245 245 245 F F F 3 060-079 137 137 137 272 272 272 F F F 4 080-099 137 137 137 272 272 272 F F F 5 100-119 126 126 126 253 253 253 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 136 136 136 271 271 271 F F F 8 160-179 134 134 134 268 268 268 F F F 9 180-199 134 134 134 268 268 268 F F F 10 200-219 133 133 133 265 265 265 F F F 11 220-239 136 136 136 271 271 271 F F F 12 240-259 136 136 136 271 271 271 F F F 13 260-279 136 136 136 271 271 271 F F F 14 280-299 134 134 134 268 268 268 F F F 15 300-319 134 134 134 268 268 268 F F F 16 320-339 137 137 137 272 272 272 F F F 17 340-359 137 137 137 272 272 272 F F F 18 360-379 136 136 136 271 271 271 F F F 19 380-399 132 132 132 263 263 263 F F F 20 400-419 134 134 134 266 266 266 F F F 21 420-439 137 137 137 272 272 272 F F F 22 440-459 134 134 134 268 268 268 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 133 133 133 265 265 265 F F F 25 500-519 134 134 134 269 269 269 F F F 26 520-539 134 134 134 267 267 267 F F F 27 540-559 134 134 134 267 267 267 F F F 28 560-579 136 136 136 271 271 271 F F F 29 580-599 136 136 136 271 271 271 F F F Max gen: 137, Max flush: 272 Begin loop 69 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.036230056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048088087Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048447501Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048488404Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048500648Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048518471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} {"msg":"Crucible e24975db-d657-477c-82ac-3c83365c7295 has session id: 4e873d51-704e-416a-a673-2afc707df06e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048881403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Upstairs opts: Upstairs UUID: e24975db-d657-477c-82ac-3c83365c7295, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.048918707Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Crucible stats registered with UUID: e24975db-d657-477c-82ac-3c83365c7295","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.04893421Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:57.048947954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049128936Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049154865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049169478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049292285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"task":"crutest"} {"msg":"The guest has requested activation with gen:138","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049310457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} {"msg":"e24975db-d657-477c-82ac-3c83365c7295 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.04932836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049341194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"0","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049355228Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"1","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049369222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"2","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049382836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.04939688Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"ds_connection connected from Ok(127.0.0.1:46650)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049421359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"2","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049435643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"ds_connection connected from Ok(127.0.0.1:57999)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049450547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"1","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"ds_connection connected from Ok(127.0.0.1:55167)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.049542317Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"io task","client":"0","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050141327Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"1","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050162808Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"0","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050367579Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"2","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050514176Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050529639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[0]R flush_numbers[0..12]: [271, 261, 245, 272, 272, 253, 254, 271, 268, 268, 265, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.05075709Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[0]R generation[0..12]: [136, 130, 121, 137, 137, 126, 127, 136, 134, 134, 133, 136]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050775942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050792245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[1]R flush_numbers[0..12]: [271, 261, 245, 272, 272, 253, 254, 271, 268, 268, 265, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050808158Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[1]R generation[0..12]: [136, 130, 121, 137, 137, 126, 127, 136, 134, 134, 133, 136]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050823751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050840004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[2]R flush_numbers[0..12]: [271, 261, 245, 272, 272, 253, 254, 271, 268, 268, 265, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050855588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[2]R generation[0..12]: [136, 130, 121, 137, 137, 126, 127, 136, 134, 134, 133, 136]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050871381Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050888014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Max found gen is 138","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050901688Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Generation requested: 138 >= found:138","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050916071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Next flush: 273","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050929326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.05094285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"":"downstairs","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:53:57.050957174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050978135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"0","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.050992828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"1","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.051006962Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"client":"2","":"downstairs","session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.051021756Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.05103528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"e24975db-d657-477c-82ac-3c83365c7295 is now active with session: 4e873d51-704e-416a-a673-2afc707df06e","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:53:57.051050363Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.051065807Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752,"session_id":"4e873d51-704e-416a-a673-2afc707df06e"} {"msg":"The guest has finished waiting for activation with:138","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.051079371Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} {"msg":"Activated sub_volume e24975db-d657-477c-82ac-3c83365c7295","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:53:57.051090996Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 548 len 24576 data: 13 11 12 9 6 6 02/30 Write block 104 len 40960 data: 14 11 11 7 9 8 9 8 7 7 03/30 Write block 375 len 40960 data: 9 10 9 11 9 12 13 13 11 10 04/30 Read block 208 len 28672 05/30 Read block 213 len 32768 06/30 Read block 367 len 24576 07/30 Write block 498 len 8192 data: 15 18 08/30 Read block 574 len 20480 09/30 Read block 341 len 4096 10/30 Read block 259 len 20480 11/30 Write block 196 len 36864 data: 14 17 17 16 15 15 18 20 20 12/30 Write block 290 len 36864 data: 14 16 14 13 12 12 14 16 14 13/30 Read block 230 len 32768 14/30 Write block 159 len 40960 data: 14 18 17 16 12 9 7 8 10 14 15/30 Read block 442 len 20480 16/30 Flush 17/30 Write block 416 len 24576 data: 8 9 10 12 10 11 18/30 Write block 41 len 12288 data: 9 10 9 19/30 Read block 29 len 4096 20/30 Write block 42 len 24576 data: 11 10 12 12 11 10 21/30 Read block 486 len 24576 22/30 Read block 261 len 4096 23/30 Read block 101 len 24576 24/30 Flush 25/30 Write block 586 len 40960 data: 11 12 11 14 15 14 13 12 11 10 26/30 Read block 48 len 40960 27/30 Read block 461 len 28672 28/30 Read block 154 len 24576 29/30 Read block 458 len 28672 30/30 Read block 540 len 4096 ---------------------------------------------------------------- Crucible gen:138 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Write 10 Done Done Done false 1003 Acked Read 7 Done Done Done false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 2 Done Done Sent false 1007 Acked Read 5 Done Done Sent false 1008 Acked Read 1 Done Done Sent false 1009 Acked Read 5 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 10 Done Done Sent false 1014 Acked Read 5 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 6 Done Done Sent false 1017 Acked Write 3 Done Done Sent false 1018 Acked Read 1 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 6 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 6 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 10 Done Done Sent false 1026 Acked Read 7 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 7 Done Done Sent false 1029 Acked Read 1 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 26 27 Done 30 29 4 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:53:57.0643248Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1752} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 271 271 271 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 138 138 121 274 274 245 F F F <--- 3 060-079 137 137 137 272 272 272 F F F 4 080-099 137 137 137 272 272 272 F F F 5 100-119 138 138 126 273 273 253 F F T <--- 6 120-139 127 127 127 254 254 254 F F F 7 140-159 138 138 136 273 273 271 F F F <--- 8 160-179 138 138 134 273 273 268 F F F <--- 9 180-199 138 138 134 273 273 268 F F F <--- 10 200-219 138 138 133 273 273 265 F F F <--- 11 220-239 136 136 136 271 271 271 F F F 12 240-259 136 136 136 271 271 271 F F F 13 260-279 136 136 136 271 271 271 F F F 14 280-299 138 138 134 273 273 268 F F F <--- 15 300-319 134 134 134 268 268 268 F F F 16 320-339 137 137 137 272 272 272 F F F 17 340-359 137 137 137 272 272 272 F F F 18 360-379 138 138 136 273 273 271 F F T <--- 19 380-399 138 138 132 273 273 263 F F T <--- 20 400-419 138 138 134 274 274 266 F F F <--- 21 420-439 138 138 137 274 274 272 F F F <--- 22 440-459 134 134 134 268 268 268 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 138 138 133 273 273 265 F F F <--- 25 500-519 134 134 134 269 269 269 F F F 26 520-539 134 134 134 267 267 267 F F F 27 540-559 138 138 134 273 273 267 F F T <--- 28 560-579 136 136 136 271 271 271 F F F 29 580-599 136 136 136 271 271 271 T T F <--- Max gen: 138, Max flush: 274 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 69 Verifying data now No change after verify Loop: 69 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 136 136 271 271 271 F F F 1 020-039 130 130 130 261 261 261 F F F 2 040-059 138 138 138 274 274 274 F F F 3 060-079 137 137 137 272 272 272 F F F 4 080-099 137 137 137 272 272 272 F F F 5 100-119 138 138 138 273 273 273 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 138 138 138 273 273 273 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 138 138 138 273 273 273 F F F 10 200-219 138 138 138 273 273 273 F F F 11 220-239 136 136 136 271 271 271 F F F 12 240-259 136 136 136 271 271 271 F F F 13 260-279 136 136 136 271 271 271 F F F 14 280-299 138 138 138 273 273 273 F F F 15 300-319 134 134 134 268 268 268 F F F 16 320-339 137 137 137 272 272 272 F F F 17 340-359 137 137 137 272 272 272 F F F 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 138 138 138 274 274 274 F F F 22 440-459 134 134 134 268 268 268 F F F 23 460-479 135 135 135 270 270 270 F F F 24 480-499 138 138 138 273 273 273 F F F 25 500-519 134 134 134 269 269 269 F F F 26 520-539 134 134 134 267 267 267 F F F 27 540-559 138 138 138 273 273 273 F F F 28 560-579 136 136 136 271 271 271 F F F 29 580-599 139 139 139 275 275 275 F F F Max gen: 139, Max flush: 275 Begin loop 70 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.391156984Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:01.40182595Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.401882366Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412114081Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412404165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412443418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412455693Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412476674Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} {"msg":"Crucible 3281a8e6-65f5-45a5-959c-b8bda14076ae has session id: b4c39258-ff4a-4f9f-96d9-0dbbad989d38","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412851712Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Upstairs opts: Upstairs UUID: 3281a8e6-65f5-45a5-959c-b8bda14076ae, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412905478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Crucible stats registered with UUID: 3281a8e6-65f5-45a5-959c-b8bda14076ae","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.412921121Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:01.412934326Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413107371Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413143645Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413159298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413287593Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"task":"crutest"} {"msg":"The guest has requested activation with gen:140","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413304515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} {"msg":"3281a8e6-65f5-45a5-959c-b8bda14076ae active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413320798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413333373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"0","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413347137Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"1","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413360921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"2","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413374435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413390858Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413405172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"ds_connection connected from Ok(127.0.0.1:42183)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413457459Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"0","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"ds_connection connected from Ok(127.0.0.1:44678)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413481718Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"2","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"ds_connection connected from Ok(127.0.0.1:41889)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.413496922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"io task","client":"1","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414229175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"2","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414272496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"1","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414345734Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"0","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414613528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414651581Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[0]R flush_numbers[0..12]: [271, 261, 274, 272, 272, 273, 254, 273, 273, 273, 273, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414713305Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[0]R generation[0..12]: [136, 130, 138, 137, 137, 138, 127, 138, 138, 138, 138, 136]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414730777Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.41474749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[1]R flush_numbers[0..12]: [271, 261, 274, 272, 272, 273, 254, 273, 273, 273, 273, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414763503Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[1]R generation[0..12]: [136, 130, 138, 137, 137, 138, 127, 138, 138, 138, 138, 136]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414779526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414798858Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[2]R flush_numbers[0..12]: [271, 261, 274, 272, 272, 273, 254, 273, 273, 273, 273, 271]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414814661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[2]R generation[0..12]: [136, 130, 138, 137, 137, 138, 127, 138, 138, 138, 138, 136]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414830524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414846737Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Max found gen is 140","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:54:01.414860071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Generation requested: 140 >= found:140","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414877304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Next flush: 276","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414890728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414903952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"":"downstairs","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414917966Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414930581Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"0","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414944475Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"1","":"downstairs","session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414958418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"client":"2","":"downstairs","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414976121Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.414989725Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"3281a8e6-65f5-45a5-959c-b8bda14076ae is now active with session: b4c39258-ff4a-4f9f-96d9-0dbbad989d38","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.415006198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.415019112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760,"session_id":"b4c39258-ff4a-4f9f-96d9-0dbbad989d38"} {"msg":"The guest has finished waiting for activation with:140","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.415032516Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} {"msg":"Activated sub_volume 3281a8e6-65f5-45a5-959c-b8bda14076ae","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:01.415044061Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 248 len 32768 data: 7 9 11 11 10 11 9 10 02/30 Write block 445 len 36864 data: 7 7 6 7 6 8 11 13 14 03/30 Read block 462 len 8192 04/30 Write block 16 len 28672 data: 13 12 11 10 9 12 12 05/30 Read block 22 len 12288 06/30 Read block 447 len 32768 07/30 Read block 275 len 12288 08/30 Write block 513 len 40960 data: 12 12 11 13 11 12 12 12 11 11 09/30 Read block 346 len 16384 10/30 Write block 11 len 16384 data: 13 12 11 13 11/30 Write block 472 len 20480 data: 12 11 10 9 3 12/30 Read block 362 len 40960 13/30 Read block 199 len 32768 14/30 Write block 234 len 40960 data: 12 12 13 13 15 14 12 11 10 11 15/30 Write block 300 len 40960 data: 12 13 9 9 8 7 5 5 6 8 16/30 Write block 199 len 8192 data: 17 16 17/30 Read block 454 len 20480 18/30 Flush 19/30 Write block 430 len 8192 data: 11 9 20/30 Write block 15 len 36864 data: 12 14 13 12 11 10 13 13 14 21/30 Read block 465 len 12288 22/30 Write block 49 len 36864 data: 12 12 12 9 8 9 10 9 7 23/30 Write block 545 len 24576 data: 10 8 11 14 12 13 24/30 Write block 488 len 40960 data: 13 15 14 13 13 12 14 16 17 16 25/30 Write block 142 len 24576 data: 11 9 10 10 9 11 26/30 Write block 12 len 24576 data: 13 12 14 13 15 14 27/30 Write block 350 len 16384 data: 16 15 15 13 28/30 Write block 95 len 20480 data: 14 13 13 14 13 29/30 Read block 517 len 8192 30/30 Write block 425 len 20480 data: 14 11 8 9 12 ---------------------------------------------------------------- Crucible gen:140 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 8 Done Done Done false 1001 Acked Write 9 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Write 7 Done Done Done false 1004 Acked Read 3 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 3 Done Done Done false 1007 Acked Write 10 Done Done Done false 1008 Acked Read 4 Done Done Done false 1009 Acked Write 4 Done Done Done false 1010 Acked Write 5 Done Done Done false 1011 Acked Read 10 Done Done Done false 1012 Acked Read 8 Done Done Done false 1013 Acked Write 10 Sent Done Done false 1014 Acked Write 10 Sent Done Done false 1015 Acked Write 2 Sent Done Done false 1016 Acked Read 5 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 2 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Read 3 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Write 6 Sent Done Done false 1023 Acked Write 10 Sent Done Done false 1024 Acked Write 6 Sent Done Done false 1025 Acked Write 6 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 5 Sent Done Done false 1028 Acked Read 2 Sent Done Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 17 1 1 19 Done 13 29 29 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:01.427994319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1760} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 136 140 140 271 276 276 T T T <--- 1 020-039 130 140 140 261 276 276 T T T <--- 2 040-059 138 138 138 274 274 274 F T T <--- 3 060-079 137 137 137 272 272 272 F F F 4 080-099 137 137 137 272 272 272 F T T <--- 5 100-119 138 138 138 273 273 273 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 138 138 138 273 273 273 F T T <--- 8 160-179 138 138 138 273 273 273 F F F 9 180-199 138 140 140 273 276 276 F F F <--- 10 200-219 138 140 140 273 276 276 F F F <--- 11 220-239 136 140 140 271 276 276 F F F <--- 12 240-259 136 140 140 271 276 276 T F F <--- 13 260-279 136 136 136 271 271 271 F F F 14 280-299 138 138 138 273 273 273 F F F 15 300-319 134 140 140 268 276 276 F F F <--- 16 320-339 137 137 137 272 272 272 F F F 17 340-359 137 137 137 272 272 272 F T T <--- 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 138 138 138 274 274 274 F T T <--- 22 440-459 134 140 140 268 276 276 T F F <--- 23 460-479 135 140 140 270 276 276 T F F <--- 24 480-499 138 138 138 273 273 273 F T T <--- 25 500-519 134 140 140 269 276 276 T F F <--- 26 520-539 134 140 140 267 276 276 T F F <--- 27 540-559 138 138 138 273 273 273 F T T <--- 28 560-579 136 136 136 271 271 271 F F F 29 580-599 139 139 139 275 275 275 F F F Max gen: 140, Max flush: 276 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 70 Verifying data now No change after verify Loop: 70 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 277 277 277 F F F 1 020-039 141 141 141 277 277 277 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 137 137 137 272 272 272 F F F 4 080-099 141 141 141 277 277 277 F F F 5 100-119 138 138 138 273 273 273 F F F 6 120-139 127 127 127 254 254 254 F F F 7 140-159 141 141 141 277 277 277 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 140 140 140 276 276 276 F F F 10 200-219 140 140 140 276 276 276 F F F 11 220-239 140 140 140 276 276 276 F F F 12 240-259 140 140 140 276 276 276 F F F 13 260-279 136 136 136 271 271 271 F F F 14 280-299 138 138 138 273 273 273 F F F 15 300-319 140 140 140 276 276 276 F F F 16 320-339 137 137 137 272 272 272 F F F 17 340-359 141 141 141 277 277 277 F F F 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 141 141 141 277 277 277 F F F 22 440-459 140 140 140 276 276 276 F F F 23 460-479 140 140 140 276 276 276 F F F 24 480-499 141 141 141 277 277 277 F F F 25 500-519 140 140 140 276 276 276 F F F 26 520-539 140 140 140 276 276 276 F F F 27 540-559 141 141 141 277 277 277 F F F 28 560-579 136 136 136 271 271 271 F F F 29 580-599 139 139 139 275 275 275 F F F Max gen: 141, Max flush: 277 Begin loop 71 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.776463665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.788371874Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.788688117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.7887292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.788741814Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.788759646Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} {"msg":"Crucible f39b8f13-5cb1-4962-af47-c35afa512761 has session id: 451db427-6c6d-4648-aafb-005234b95a35","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789156974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Upstairs opts: Upstairs UUID: f39b8f13-5cb1-4962-af47-c35afa512761, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789191869Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Crucible stats registered with UUID: f39b8f13-5cb1-4962-af47-c35afa512761","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789207722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:05.789220787Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789413073Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789449647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789465331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789564078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"task":"crutest"} {"msg":"The guest has requested activation with gen:142","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789579991Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} {"msg":"f39b8f13-5cb1-4962-af47-c35afa512761 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789597993Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789610907Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"0","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789624821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"1","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789638575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"2","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789652499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789666813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789681207Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"ds_connection connected from Ok(127.0.0.1:59879)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.78976655Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"0","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"ds_connection connected from Ok(127.0.0.1:39468)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.789841148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"1","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"ds_connection connected from Ok(127.0.0.1:63733)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.78985871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"io task","client":"2","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790476033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"0","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790507119Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"1","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790524462Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"2","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790835757Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.79087465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[0]R flush_numbers[0..12]: [277, 277, 277, 272, 277, 273, 254, 277, 273, 276, 276, 276]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790934004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[0]R generation[0..12]: [141, 141, 141, 137, 141, 138, 127, 141, 138, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790951736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790968499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[1]R flush_numbers[0..12]: [277, 277, 277, 272, 277, 273, 254, 277, 273, 276, 276, 276]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.790984522Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[1]R generation[0..12]: [141, 141, 141, 137, 141, 138, 127, 141, 138, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791000595Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791016938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[2]R flush_numbers[0..12]: [277, 277, 277, 272, 277, 273, 254, 277, 273, 276, 276, 276]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791032871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[2]R generation[0..12]: [141, 141, 141, 137, 141, 138, 127, 141, 138, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791048494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791064947Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 142","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791087078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Generation requested: 142 >= found:142","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791101591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Next flush: 278","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791114806Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.79112827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"":"downstairs","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791142484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791155308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"0","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Transition from WaitQuorum to Active","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791172411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"1","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791186784Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"client":"2","":"downstairs","session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791200818Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791214123Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"f39b8f13-5cb1-4962-af47-c35afa512761 is now active with session: 451db427-6c6d-4648-aafb-005234b95a35","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791229206Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791242061Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768,"session_id":"451db427-6c6d-4648-aafb-005234b95a35"} {"msg":"The guest has finished waiting for activation with:142","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.791255245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} {"msg":"Activated sub_volume f39b8f13-5cb1-4962-af47-c35afa512761","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:05.79126727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 532 len 24576 data: 12 10 9 7 7 6 02/30 Read block 371 len 28672 03/30 Read block 468 len 32768 04/30 Write block 64 len 36864 data: 9 9 9 8 7 6 6 6 7 05/30 Read block 155 len 8192 06/30 Write block 573 len 8192 data: 9 10 07/30 Read block 343 len 20480 08/30 Flush 09/30 Write block 485 len 24576 data: 12 12 11 14 16 15 10/30 Write block 125 len 12288 data: 15 13 11 11/30 Write block 560 len 40960 data: 12 12 12 11 10 11 10 9 11 13 12/30 Read block 422 len 28672 13/30 Read block 388 len 20480 14/30 Write block 490 len 8192 data: 16 14 15/30 Write block 144 len 16384 data: 11 11 10 12 16/30 Read block 379 len 4096 17/30 Read block 167 len 24576 18/30 Flush 19/30 Write block 136 len 24576 data: 14 11 10 10 10 11 20/30 Flush 21/30 Write block 321 len 16384 data: 14 11 10 11 22/30 Write block 356 len 4096 data: 16 23/30 Read block 81 len 20480 24/30 Read block 322 len 36864 25/30 Write block 132 len 40960 data: 12 12 13 12 15 12 11 11 11 12 26/30 Read block 331 len 8192 27/30 Write block 26 len 4096 data: 11 28/30 Read block 200 len 28672 29/30 Read block 55 len 32768 30/30 Write block 1 len 24576 data: 3 5 8 11 13 13 ---------------------------------------------------------------- Crucible gen:142 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 6 Done Done Done false 1001 Acked Read 7 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 9 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Write 2 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Flush 0 Done Done Sent false 1008 Acked Write 6 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Write 10 Done Done Sent false 1011 Acked Read 7 Done Done Sent false 1012 Acked Read 5 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 1 Done Done Sent false 1016 Acked Read 6 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 4 Done Done Sent false 1021 Acked Write 1 Done Done Sent false 1022 Acked Read 5 Done Done Sent false 1023 Acked Read 9 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 2 Done Done Sent false 1026 Acked Write 1 Done Done Sent false 1027 Acked Read 7 Done Done Sent false 1028 Acked Read 8 Done Done Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 25 27 Done 29 29 5 63 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) Some(JobId(1019)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:05.803576625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1768} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 141 141 141 277 277 277 T T F <--- 1 020-039 141 141 141 277 277 277 T T F <--- 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 137 278 278 272 F F T <--- 4 080-099 141 141 141 277 277 277 F F F 5 100-119 138 138 138 273 273 273 F F F 6 120-139 142 142 127 280 280 254 T T F <--- 7 140-159 142 142 141 280 280 277 T T F <--- 8 160-179 138 138 138 273 273 273 F F F 9 180-199 140 140 140 276 276 276 F F F 10 200-219 140 140 140 276 276 276 F F F 11 220-239 140 140 140 276 276 276 F F F 12 240-259 140 140 140 276 276 276 F F F 13 260-279 136 136 136 271 271 271 F F F 14 280-299 138 138 138 273 273 273 F F F 15 300-319 140 140 140 276 276 276 F F F 16 320-339 137 137 137 272 272 272 T T F <--- 17 340-359 141 141 141 277 277 277 T T F <--- 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 141 141 141 277 277 277 F F F 22 440-459 140 140 140 276 276 276 F F F 23 460-479 140 140 140 276 276 276 F F F 24 480-499 142 142 141 279 279 277 F F F <--- 25 500-519 140 140 140 276 276 276 F F F 26 520-539 142 142 140 278 278 276 F F T <--- 27 540-559 141 141 141 277 277 277 F F F 28 560-579 142 142 136 279 279 271 F F F <--- 29 580-599 139 139 139 275 275 275 F F F Max gen: 142, Max flush: 280 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 71 Verifying data now No change after verify Loop: 71 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 281 281 281 F F F 1 020-039 143 143 143 281 281 281 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 141 141 141 277 277 277 F F F 5 100-119 138 138 138 273 273 273 F F F 6 120-139 143 143 143 281 281 281 F F F 7 140-159 143 143 143 281 281 281 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 140 140 140 276 276 276 F F F 10 200-219 140 140 140 276 276 276 F F F 11 220-239 140 140 140 276 276 276 F F F 12 240-259 140 140 140 276 276 276 F F F 13 260-279 136 136 136 271 271 271 F F F 14 280-299 138 138 138 273 273 273 F F F 15 300-319 140 140 140 276 276 276 F F F 16 320-339 143 143 143 281 281 281 F F F 17 340-359 143 143 143 281 281 281 F F F 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 141 141 141 277 277 277 F F F 22 440-459 140 140 140 276 276 276 F F F 23 460-479 140 140 140 276 276 276 F F F 24 480-499 142 142 142 279 279 279 F F F 25 500-519 140 140 140 276 276 276 F F F 26 520-539 142 142 142 278 278 278 F F F 27 540-559 141 141 141 277 277 277 F F F 28 560-579 142 142 142 279 279 279 F F F 29 580-599 139 139 139 275 275 275 F F F Max gen: 143, Max flush: 281 Begin loop 72 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.124171181Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:10.134846345Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.13490361Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145236292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145551845Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145588839Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145601633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145619366Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} {"msg":"Crucible 057686ed-09ba-46f4-b15b-af34e41fb3db has session id: 3b34acb5-a2d4-459a-a172-e28fcb9784cc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145949383Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Upstairs opts: Upstairs UUID: 057686ed-09ba-46f4-b15b-af34e41fb3db, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.145999701Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Crucible stats registered with UUID: 057686ed-09ba-46f4-b15b-af34e41fb3db","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146016034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:10.146029238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146199324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146223674Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146238337Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146380946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"task":"crutest"} {"msg":"The guest has requested activation with gen:144","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146398688Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} {"msg":"057686ed-09ba-46f4-b15b-af34e41fb3db active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146414551Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146437571Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"0","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146451775Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"1","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146467008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"2","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146480542Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146524283Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146547243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:41908)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146562867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"1","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:37035)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146578Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"2","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"ds_connection connected from Ok(127.0.0.1:57479)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.146604968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"io task","client":"0","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147324067Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"2","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147353684Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"1","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147370767Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"0","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147697645Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147714028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 273, 281, 281, 273, 276, 276, 276]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147739527Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[0]R generation[0..12]: [143, 143, 141, 142, 141, 138, 143, 143, 138, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147758119Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147774961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 273, 281, 281, 273, 276, 276, 276]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147791264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[1]R generation[0..12]: [143, 143, 141, 142, 141, 138, 143, 143, 138, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147807157Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.14782349Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 273, 281, 281, 273, 276, 276, 276]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147839663Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[2]R generation[0..12]: [143, 143, 141, 142, 141, 138, 143, 143, 138, 140, 140, 140]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147855706Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147872029Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Max found gen is 144","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147885793Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid"Wait for a query_work_queue command to finish before sending IO :1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Generation requested: 144 >= found:144","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147908284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Next flush: 282","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147921908Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147935572Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"":"downstairs","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147949866Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147963959Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"0","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147980882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":"1","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.147994756Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"client":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 "2","":"downstairs","session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.148012408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.148026033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"057686ed-09ba-46f4-b15b-af34e41fb3db is now active with session: 3b34acb5-a2d4-459a-a172-e28fcb9784cc","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.148040786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.148053511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776,"session_id":"3b34acb5-a2d4-459a-a172-e28fcb9784cc"} {"msg":"The guest has finished waiting for activation with:144","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.148066765Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} {"msg":"Activated sub_volume 057686ed-09ba-46f4-b15b-af34e41fb3db","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:10.14807879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 20480 data: 9 12 14 15 14 02/30 Read block 269 len 28672 03/30 Flush 04/30 Write block 500 len 12288 data: 13 14 15 05/30 Read block 332 len 40960 06/30 Write block 351 len 4096 data: 16 07/30 Write block 505 len 32768 data: 14 16 17 16 15 17 17 16 08/30 Read block 239 len 12288 09/30 Read block 215 len 40960 10/30 Read block 360 len 32768 11/30 Write block 271 len 12288 data: 12 14 14 12/30 Write block 210 len 40960 data: 7 7 8 8 7 6 5 5 6 8 13/30 Write block 473 len 24576 data: 12 11 10 4 5 10 14/30 Read block 347 len 40960 15/30 Write block 242 len 28672 data: 11 12 11 9 8 6 8 16/30 Write block 109 len 20480 data: 9 10 9 8 8 17/30 Write block 157 len 8192 data: 10 12 18/30 Write block 536 len 20480 data: 8 7 6 7 8 19/30 Write block 306 len 28672 data: 6 6 7 9 7 9 9 20/30 Read block 584 len 32768 21/30 Write block 584 len 32768 data: 9 10 12 13 12 15 16 15 22/30 Flush 23/30 Write block 230 len 20480 data: 7 6 9 11 13 24/30 Read block 334 len 36864 25/30 Read block 532 len 4096 26/30 Read block 400 len 28672 27/30 Write block 152 len 8192 data: 9 8 28/30 Read block 394 len 40960 29/30 Write block 485 len 16384 data: 13 13 12 15 30/30 Write block 336 len 16384 data: 15 14 16 13 ---------------------------------------------------------------- Crucible gen:144 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Read 7 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Write 3 Sent Done Done false 1004 Acked Read 10 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Read 10 Sent Done Done false 1009 Acked Read 8 Sent Done Done false 1010 Acked Write 3 Sent Done Done false 1011 Acked Write 10 Sent Done Done false 1012 Acked Write 6 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Write 5 Sent Done Done false 1016 Acked Write 2 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Write 7 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 8 Sent Done Done false 1021 Acked Flush 0 Sent Done Done false 1022 Acked Write 5 Sent Done Done false 1023 Acked Read 9 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Read 7 Sent Done Done false 1026 Acked Write 2 Sent Done Done false 1027 Acked Read 10 Sent Done Done false 1028 Acked Write 4 Sent Sent Sent false 1029 Acked Write 4 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 29 2 2 33 Done 1 28 28 57 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1021)) Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:10.160659997Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1776} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 281 281 281 F F F 1 020-039 143 143 143 281 281 281 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 141 141 141 277 277 277 F F F 5 100-119 138 144 144 273 283 283 F F F <--- 6 120-139 143 143 143 281 281 281 F F F 7 140-159 143 144 144 281 283 283 F T T <--- 8 160-179 138 138 138 273 273 273 F F F 9 180-199 140 140 140 276 276 276 F F F 10 200-219 140 144 144 276 283 283 F F F <--- 11 220-239 140 140 140 276 276 276 F T T <--- 12 240-259 140 144 144 276 283 283 F F F <--- 13 260-279 136 144 144 271 283 283 F F F <--- 14 280-299 138 138 138 273 273 273 F F F 15 300-319 140 144 144 276 283 283 F F F <--- 16 320-339 143 143 143 281 281 281 F T T <--- 17 340-359 143 144 144 281 283 283 F F F <--- 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 141 141 141 277 277 277 F F F 22 440-459 140 144 144 276 282 282 T F F <--- 23 460-479 140 144 144 276 283 283 F F F <--- 24 480-499 142 142 142 279 279 279 F T T <--- 25 500-519 140 144 144 276 283 283 F F F <--- 26 520-539 142 144 144 278 283 283 F F F <--- 27 540-559 141 144 144 277 283 283 F F F <--- 28 560-579 142 142 142 279 279 279 F F F 29 580-599 139 144 144 275 283 283 F F F <--- Max gen: 144, Max flush: 283 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 72 Verifying data now No change after verify Loop: 72 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 281 281 281 F F F 1 020-039 143 143 143 281 281 281 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 141 141 141 277 277 277 F F F 5 100-119 144 144 144 283 283 283 F F F 6 120-139 143 143 143 281 281 281 F F F 7 140-159 145 145 145 284 284 284 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 140 140 140 276 276 276 F F F 10 200-219 144 144 144 283 283 283 F F F 11 220-239 145 145 145 284 284 284 F F F 12 240-259 144 144 144 283 283 283 F F F 13 260-279 144 144 144 283 283 283 F F F 14 280-299 138 138 138 273 273 273 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 145 145 145 284 284 284 F F F 17 340-359 144 144 144 283 283 283 F F F 18 360-379 138 138 138 273 273 273 F F F 19 380-399 138 138 138 273 273 273 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 141 141 141 277 277 277 F F F 22 440-459 144 144 144 282 282 282 F F F 23 460-479 144 144 144 283 283 283 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 144 144 144 283 283 283 F F F 27 540-559 144 144 144 283 283 283 F F F 28 560-579 142 142 142 279 279 279 F F F 29 580-599 144 144 144 283 283 283 F F F Max gen: 145, Max flush: 284 Begin loop 73 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.499774377Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.511684475Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51204139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512079584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512092498Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512110221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} {"msg":"Crucible 5c309147-abdb-4ed4-a410-8c6a1e967f5b has session id: caa2bfd6-73e6-4d6e-a304-e286e1428e72","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512454322Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Upstairs opts: Upstairs UUID: 5c309147-abdb-4ed4-a410-8c6a1e967f5b, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512484558Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Crucible stats registered with UUID: 5c309147-abdb-4ed4-a410-8c6a1e967f5b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512505229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:14.512518564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512755871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512791695Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512806889Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512934194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"task":"crutest"} {"msg":"The guest has requested activation with gen:146","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.512948847Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} {"msg":"5c309147-abdb-4ed4-a410-8c6a1e967f5b active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513017797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513032301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"0","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513046435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"1","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513060589Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"2","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513094034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513130518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513154618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"ds_connection connected from Ok(127.0.0.1:42363)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513170221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"2","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"ds_connection connected from Ok(127.0.0.1:54806)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513188413Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"1","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"ds_connection connected from Ok(127.0.0.1:33377)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513223238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"io task","client":"0","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513895317Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"0","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.513918547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"1","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51393494Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"2","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514271294Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514317274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 283, 281, 284, 273, 276, 283, 284]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514354788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[0]R generation[0..12]: [143, 143, 141, 142, 141, 144, 143, 145, 138, 140, 144, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51437201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514388433Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 283, 281, 284, 273, 276, 283, 284]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514404086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[1]R generation[0..12]: [143, 143, 141, 142, 141, 144, 143, 145, 138, 140, 144, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51441955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514435863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 283, 281, 284, 273, 276, 283, 284]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514451626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[2]R generation[0..12]: [143, 143, 141, 142, 141, 144, 143, 145, 138, 140, 144, 145]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514469768Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514485981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id"Wait for a query_work_queue command to finish before sending IO :"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Max found gen is 146","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514505882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Generation requested: 146 >= found:146","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514520256Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Next flush: 285","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51453368Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514546825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"":"downstairs","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514561168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514573693Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"0","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:54:14.514588107Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"1","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51460418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"client":"2","":"downstairs","session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514618424Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514631898Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"5c309147-abdb-4ed4-a410-8c6a1e967f5b is now active with session: caa2bfd6-73e6-4d6e-a304-e286e1428e72","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514646801Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514659386Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784,"session_id":"caa2bfd6-73e6-4d6e-a304-e286e1428e72"} {"msg":"The guest has finished waiting for activation with:146","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.51467254Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} {"msg":"Activated sub_volume 5c309147-abdb-4ed4-a410-8c6a1e967f5b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:14.514685445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 279 len 24576 data: 9 7 5 9 8 12 02/30 Write block 572 len 20480 data: 12 10 11 9 7 03/30 Write block 285 len 24576 data: 18 20 20 17 16 15 04/30 Read block 430 len 24576 05/30 Flush 06/30 Write block 223 len 20480 data: 14 12 7 6 7 07/30 Write block 186 len 8192 data: 14 12 08/30 Read block 187 len 8192 09/30 Read block 359 len 24576 10/30 Write block 468 len 4096 data: 15 11/30 Write block 590 len 8192 data: 17 16 12/30 Write block 540 len 32768 data: 9 9 9 8 8 11 9 12 13/30 Write block 545 len 4096 data: 12 14/30 Read block 526 len 4096 15/30 Write block 428 len 20480 data: 10 13 12 10 6 16/30 Write block 382 len 8192 data: 14 12 17/30 Read block 201 len 28672 18/30 Write block 255 len 16384 data: 11 10 13 13 19/30 Read block 158 len 20480 20/30 Write block 208 len 8192 data: 9 9 21/30 Write block 138 len 20480 data: 12 12 12 13 12 22/30 Read block 202 len 36864 23/30 Write block 379 len 8192 data: 10 13 24/30 Read block 101 len 4096 25/30 Write block 241 len 4096 data: 12 26/30 Write block 154 len 16384 data: 11 13 12 11 27/30 Read block 221 len 8192 28/30 Read block 524 len 20480 29/30 Write block 264 len 16384 data: 12 13 16 14 30/30 Read block 545 len 12288 ---------------------------------------------------------------- Crucible gen:146 GIO:true work queues: Upstairs:0 downstairs:25 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1005 Acked Write 5 Done Done Done false 1006 Acked Write 2 Done Done Done false 1007 Acked Read 2 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Write 1 Done Done Sent false 1010 Acked Write 2 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Read 1 Done Done Sent false 1014 Acked Write 5 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Write 5 Done Done Sent false 1021 Acked Read 9 Done Done Sent false 1022 Acked Write 2 Done Done Sent false 1023 Acked Read 1 Done Done Sent false 1024 Acked Write 1 Done Done Sent false 1025 Acked Write 4 Done Done Sent false 1026 Acked Read 2 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 3 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 23 24 Done 24 25 2 51 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1004)) Some(JobId(1004)) Some(JobId(1004)) Downstairs last five completed: 1004 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:14.524671247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1784} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 281 281 281 F F F 1 020-039 143 143 143 281 281 281 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 141 141 141 277 277 277 F F F 5 100-119 144 144 144 283 283 283 F F F 6 120-139 143 143 143 281 281 281 T T F <--- 7 140-159 145 145 145 284 284 284 T T F <--- 8 160-179 138 138 138 273 273 273 F F F 9 180-199 140 140 140 276 276 276 T T T 10 200-219 144 144 144 283 283 283 T T F <--- 11 220-239 145 145 145 284 284 284 T T T 12 240-259 144 144 144 283 283 283 T T F <--- 13 260-279 146 146 146 285 285 285 T T F <--- 14 280-299 146 146 146 285 285 285 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 145 145 145 284 284 284 F F F 17 340-359 144 144 144 283 283 283 F F F 18 360-379 138 138 138 273 273 273 T T F <--- 19 380-399 138 138 138 273 273 273 T T F <--- 20 400-419 138 138 138 274 274 274 F F F 21 420-439 141 141 141 277 277 277 T T F <--- 22 440-459 144 144 144 282 282 282 F F F 23 460-479 144 144 144 283 283 283 T T F <--- 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 144 144 144 283 283 283 F F F 27 540-559 144 144 144 283 283 283 T T F <--- 28 560-579 146 146 146 285 285 285 F F F 29 580-599 144 144 144 283 283 283 T T F <--- Max gen: 146, Max flush: 285 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 73 Verifying data now No change after verify Loop: 73 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 143 143 143 281 281 281 F F F 1 020-039 143 143 143 281 281 281 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 141 141 141 277 277 277 F F F 5 100-119 144 144 144 283 283 283 F F F 6 120-139 147 147 147 286 286 286 F F F 7 140-159 147 147 147 286 286 286 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 147 147 147 286 286 286 F F F 10 200-219 147 147 147 286 286 286 F F F 11 220-239 147 147 147 286 286 286 F F F 12 240-259 147 147 147 286 286 286 F F F 13 260-279 147 147 147 286 286 286 F F F 14 280-299 146 146 146 285 285 285 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 145 145 145 284 284 284 F F F 17 340-359 144 144 144 283 283 283 F F F 18 360-379 147 147 147 286 286 286 F F F 19 380-399 147 147 147 286 286 286 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 147 147 147 286 286 286 F F F 22 440-459 144 144 144 282 282 282 F F F 23 460-479 147 147 147 286 286 286 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 144 144 144 283 283 283 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 146 146 146 285 285 285 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 147, Max flush: 286 Begin loop 74 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.854943699Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.866652854Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.866972485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867014767Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867027841Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867045604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} {"msg":"Crucible 54b6a3d6-c74c-4321-8c34-e2b6ebffa072 has session id: 7d0b0076-290e-45c8-8b32-d94d23114876","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867350891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Upstairs opts: Upstairs UUID: 54b6a3d6-c74c-4321-8c34-e2b6ebffa072, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867404708Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Crucible stats registered with UUID: 54b6a3d6-c74c-4321-8c34-e2b6ebffa072","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867421071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:18.867434295Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867624293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867661007Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.86767645Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867788952Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"task":"crutest"} {"msg":"The guest has requested activation with gen:148","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867806614Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} {"msg":"54b6a3d6-c74c-4321-8c34-e2b6ebffa072 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867824376Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867837371Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"0","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867851604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"1","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867865548Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"2","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867879392Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867893846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.86790787Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"ds_connection connected from Ok(127.0.0.1:64412)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867922654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"0","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"ds_connection connected from Ok(127.0.0.1:47880)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867948702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"1","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"ds_connection connected from Ok(127.0.0.1:44621)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.867963906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"io task","client":"2","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.86871553Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"2","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.868743298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"0","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.868759911Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"1","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869112528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869129251Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[0]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 283, 286, 286, 273, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869155299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[0]R generation[0..12]: [143, 143, 141, 142, 141, 144, 147, 147, 138, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869171472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869188025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[1]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 283, 286, 286, 273, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869203778Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[1]R generation[0..12]: [143, 143, 141, 142, 141, 144, 147, 147, 138, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869219072Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869235095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[2]R flush_numbers[0..12]: [281, 281, 277, 278, 277, 283, 286, 286, 273, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869250738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[2]R generation[0..12]: [143, 143, 141, 142, 141, 144, 147, 147, 138, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869266431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869282604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Max found gen is 148","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869296328Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Generation requested: 148 >= found:148","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869310242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Next flush: 287","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869323916Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869337251Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"":"downstairs","":"downstairs","session_id":"Wait for a query_work_queue command to finish before sending IO 7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869357492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869372056Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"0","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869386029Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"1","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869399923Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"client":"2","":"downstairs","session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869414027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869427471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"54b6a3d6-c74c-4321-8c34-e2b6ebffa072 is now active with session: 7d0b0076-290e-45c8-8b32-d94d23114876","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869442455Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869457518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792,"session_id":"7d0b0076-290e-45c8-8b32-d94d23114876"} {"msg":"The guest has finished waiting for activation with:148","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869471113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} {"msg":"Activated sub_volume 54b6a3d6-c74c-4321-8c34-e2b6ebffa072","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:18.869482728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 572 len 8192 data: 13 11 03/30 Read block 1 len 32768 04/30 Write block 149 len 4096 data: 12 05/30 Read block 55 len 28672 06/30 Write block 368 len 12288 data: 10 9 11 07/30 Read block 358 len 20480 08/30 Flush 09/30 Write block 474 len 4096 data: 12 10/30 Flush 11/30 Write block 17 len 32768 data: 15 13 12 11 14 14 15 13 12/30 Read block 4 len 20480 13/30 Read block 199 len 24576 14/30 Read block 23 len 24576 15/30 Write block 129 len 4096 data: 9 16/30 Write block 456 len 4096 data: 14 17/30 Read block 7 len 16384 18/30 Flush 19/30 Flush 20/30 Write block 273 len 8192 data: 15 13 21/30 Write block 102 len 4096 data: 13 22/30 Read block 467 len 36864 23/30 Read block 431 len 4096 24/30 Read block 420 len 36864 25/30 Flush 26/30 Write block 529 len 28672 data: 12 10 12 13 11 10 8 27/30 Write block 375 len 24576 data: 10 11 10 12 11 14 28/30 Read block 367 len 16384 29/30 Write block 89 len 28672 data: 8 8 8 13 14 13 15 30/30 Write block 322 len 12288 data: 12 11 12 ---------------------------------------------------------------- Crucible gen:148 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 2 Done Sent Done false 1002 Acked Read 8 Done Sent Done false 1003 Acked Write 1 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Read 5 Done Sent Done false 1007 Acked Flush 0 Done Sent Done false 1008 Acked Write 1 Done Sent Done false 1009 Acked Flush 0 Done Sent Done false 1010 Acked Write 8 Done Sent Done false 1011 Acked Read 5 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Write 1 Done Sent Done false 1016 Acked Read 4 Done Sent Done false 1017 Acked Flush 0 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Write 1 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 1 Done Sent Done false 1023 Acked Read 9 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 30 2 34 Done 28 0 28 56 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:18.880370529Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1792} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 143 148 290 281 290 F F F <--- 1 020-039 148 143 148 290 281 290 F F F <--- 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 141 141 141 277 277 277 T F T <--- 5 100-119 148 144 148 292 283 292 F F F <--- 6 120-139 148 147 148 290 286 290 F F F <--- 7 140-159 148 147 148 288 286 288 F F F <--- 8 160-179 138 138 138 273 273 273 F F F 9 180-199 147 147 147 286 286 286 F F F 10 200-219 147 147 147 286 286 286 F F F 11 220-239 147 147 147 286 286 286 F F F 12 240-259 147 147 147 286 286 286 F F F 13 260-279 148 147 148 292 286 292 F F F <--- 14 280-299 146 146 146 285 285 285 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 145 145 145 284 284 284 T F T <--- 17 340-359 144 144 144 283 283 283 F F F 18 360-379 148 147 148 288 286 288 T F T <--- 19 380-399 147 147 147 286 286 286 T F T <--- 20 400-419 138 138 138 274 274 274 F F F 21 420-439 147 147 147 286 286 286 F F F 22 440-459 148 144 148 290 282 290 F F F <--- 23 460-479 148 147 148 289 286 289 F F F <--- 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 144 144 144 283 283 283 T F T <--- 27 540-559 147 147 147 286 286 286 F F F 28 560-579 148 146 148 288 285 288 F F F <--- 29 580-599 147 147 147 286 286 286 F F F Max gen: 148, Max flush: 292 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 74 Verifying data now No change after verify Loop: 74 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 290 290 290 F F F 1 020-039 148 148 148 290 290 290 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 149 149 149 293 293 293 F F F 5 100-119 148 148 148 292 292 292 F F F 6 120-139 148 148 148 290 290 290 F F F 7 140-159 148 148 148 288 288 288 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 147 147 147 286 286 286 F F F 10 200-219 147 147 147 286 286 286 F F F 11 220-239 147 147 147 286 286 286 F F F 12 240-259 147 147 147 286 286 286 F F F 13 260-279 148 148 148 292 292 292 F F F 14 280-299 146 146 146 285 285 285 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 149 149 149 293 293 293 F F F 17 340-359 144 144 144 283 283 283 F F F 18 360-379 149 149 149 293 293 293 F F F 19 380-399 149 149 149 293 293 293 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 147 147 147 286 286 286 F F F 22 440-459 148 148 148 290 290 290 F F F 23 460-479 148 148 148 289 289 289 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 149 149 149 293 293 293 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 148 148 148 288 288 288 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 149, Max flush: 293 Begin loop 75 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.204819566Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.216516716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.216852811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.216892004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.216904048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.216921991Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} {"msg":"Crucible 1f8474a1-394e-4168-8c27-bd11e211e9cf has session id: 5814a6d8-1fc6-4f8a-af1c-216bd2369c76","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217256456Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Upstairs opts: Upstairs UUID: 1f8474a1-394e-4168-8c27-bd11e211e9cf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217330464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Crucible stats registered with UUID: 1f8474a1-394e-4168-8c27-bd11e211e9cf","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217358381Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:23.217372655Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217538363Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217575677Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.21759149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217682581Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"task":"crutest"} {"msg":"The guest has requested activation with gen:150","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217700643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} {"msg":"1f8474a1-394e-4168-8c27-bd11e211e9cf active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.21773131Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217745384Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"0","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217759618Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"1","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217773721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"2","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217787445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217801739Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217883314Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"ds_connection connected from Ok(127.0.0.1:63777)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217907523Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"2","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"ds_connection connected from Ok(127.0.0.1:61506)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217925955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"1","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"ds_connection connected from Ok(127.0.0.1:40478)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.217943458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"io task","client":"0","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.218628431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"2","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.218664275Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"0","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.218684937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"1","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.21902188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219067811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 277, 278, 293, 292, 290, 288, 273, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219126085Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[0]R generation[0..12]: [148, 148, 141, 142, 149, 148, 148, 148, 138, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219143408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219159721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 277, 278, 293, 292, 290, 288, 273, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219175574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[1]R generation[0..12]: [148, 148, 141, 142, 149, 148, 148, 148, 138, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219191497Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.21920786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 277, 278, 293, 292, 290, 288, 273, 286, 286, 286]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219223403Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[2]R generation[0..12]: [148, 148, 141, 142, 149, 148, 148, 148, 138, 147, 147, 147]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219241215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219257728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Max found gen is 150","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219271372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Generation requested: 150 >= found:150","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219285486Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"Wait for a query_work_queue command to finish before sending IO } {"msg":"Next flush: 294","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219304678Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219318332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"":"downstairs","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219332356Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.21934493Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"0","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:54:23.219358645Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"1","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219375107Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"client":"2","":"downstairs","session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219389371Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219402406Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"1f8474a1-394e-4168-8c27-bd11e211e9cf is now active with session: 5814a6d8-1fc6-4f8a-af1c-216bd2369c76","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219417059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219429394Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800,"session_id":"5814a6d8-1fc6-4f8a-af1c-216bd2369c76"} {"msg":"The guest has finished waiting for activation with:150","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219442638Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} {"msg":"Activated sub_volume 1f8474a1-394e-4168-8c27-bd11e211e9cf","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:23.219455802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 290 len 4096 data: 16 02/30 Read block 92 len 16384 03/30 Flush 04/30 Write block 434 len 28672 data: 6 8 9 11 13 13 12 05/30 Read block 80 len 28672 06/30 Write block 246 len 12288 data: 9 7 9 07/30 Write block 231 len 28672 data: 7 10 12 14 13 14 14 08/30 Read block 153 len 40960 09/30 Read block 0 len 36864 10/30 Read block 331 len 28672 11/30 Read block 71 len 8192 12/30 Write block 353 len 20480 data: 14 13 14 17 17 13/30 Flush 14/30 Write block 106 len 24576 data: 12 8 10 10 11 10 15/30 Read block 538 len 4096 16/30 Write block 268 len 20480 data: 13 12 11 13 15 17/30 Write block 437 len 20480 data: 12 14 14 13 11 18/30 Read block 366 len 24576 19/30 Flush 20/30 Write block 214 len 8192 data: 8 7 21/30 Flush 22/30 Write block 100 len 28672 data: 17 16 14 13 15 12 13 23/30 Read block 513 len 20480 24/30 Write block 380 len 12288 data: 15 14 15 25/30 Write block 420 len 8192 data: 11 12 26/30 Write block 230 len 32768 data: 8 8 11 13 15 14 15 15 27/30 Write block 326 len 32768 data: 13 14 15 11 12 12 13 11 28/30 Read block 314 len 12288 29/30 Write block 569 len 8192 data: 14 11 30/30 Read block 586 len 40960 ---------------------------------------------------------------- Crucible gen:150 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Read 4 Done Sent Done false 1002 Acked Flush 0 Done Sent Done false 1003 Acked Write 7 Done Sent Done false 1004 Acked Read 7 Done Sent Done false 1005 Acked Write 3 Done Sent Done false 1006 Acked Write 7 Done Sent Done false 1007 Acked Read 10 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Read 7 Done Sent Done false 1010 Acked Read 2 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 6 Done Sent Done false 1014 Acked Read 1 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Write 5 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Flush 0 Done Sent Done false 1019 Acked Write 2 Done Sent Done false 1020 Acked Flush 0 Done Sent Done false 1021 Acked Write 7 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 2 Done Sent Done false 1025 Acked Write 8 Done Sent Done false 1026 Acked Write 8 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 10 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 29 1 30 Done 30 1 29 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1020)) None Some(JobId(1020)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:23.231746086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1800} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 290 290 290 F F F 1 020-039 148 148 148 290 290 290 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 149 149 149 293 293 293 F F F 5 100-119 150 148 150 296 292 296 T F T <--- 6 120-139 148 148 148 290 290 290 F F F 7 140-159 148 148 148 288 288 288 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 147 147 147 286 286 286 F F F 10 200-219 150 147 150 297 286 297 F F F <--- 11 220-239 150 147 150 295 286 295 T F T <--- 12 240-259 150 147 150 295 286 295 F F F <--- 13 260-279 150 148 150 296 292 296 F F F <--- 14 280-299 150 146 150 294 285 294 F T F <--- 15 300-319 144 144 144 283 283 283 F F F 16 320-339 149 149 149 293 293 293 T F T <--- 17 340-359 150 144 150 295 283 295 F F F <--- 18 360-379 149 149 149 293 293 293 F F F 19 380-399 149 149 149 293 293 293 T F T <--- 20 400-419 138 138 138 274 274 274 F F F 21 420-439 150 147 150 296 286 296 T F T <--- 22 440-459 150 148 150 296 290 296 F F F <--- 23 460-479 148 148 148 289 289 289 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 149 149 149 293 293 293 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 148 148 148 288 288 288 T F T <--- 29 580-599 147 147 147 286 286 286 F F F Max gen: 150, Max flush: 297 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 75 Verifying data now No change after verify Loop: 75 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 148 148 148 290 290 290 F F F 1 020-039 148 148 148 290 290 290 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 149 149 149 293 293 293 F F F 5 100-119 151 151 151 298 298 298 F F F 6 120-139 148 148 148 290 290 290 F F F 7 140-159 148 148 148 288 288 288 F F F 8 160-179 138 138 138 273 273 273 F F F 9 180-199 147 147 147 286 286 286 F F F 10 200-219 150 150 150 297 297 297 F F F 11 220-239 151 151 151 298 298 298 F F F 12 240-259 150 150 150 295 295 295 F F F 13 260-279 150 150 150 296 296 296 F F F 14 280-299 150 150 150 294 294 294 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 151 151 151 298 298 298 F F F 17 340-359 150 150 150 295 295 295 F F F 18 360-379 149 149 149 293 293 293 F F F 19 380-399 151 151 151 298 298 298 F F F 20 400-419 138 138 138 274 274 274 F F F 21 420-439 151 151 151 298 298 298 F F F 22 440-459 150 150 150 296 296 296 F F F 23 460-479 148 148 148 289 289 289 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 149 149 149 293 293 293 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 151 151 151 298 298 298 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 151, Max flush: 298 Begin loop 76 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.555409842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567118007Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567437249Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567474183Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567486798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.56750474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} {"msg":"Crucible 0fe25c58-4403-46d6-8c3d-d5410bb5af5c has session id: a80b9516-1ed6-4fba-9481-a59374580809","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567858476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Upstairs opts: Upstairs UUID: 0fe25c58-4403-46d6-8c3d-d5410bb5af5c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567929126Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Crucible stats registered with UUID: 0fe25c58-4403-46d6-8c3d-d5410bb5af5c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.567946218Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:27.567959443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568148921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568183086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568198749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568326234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"task":"crutest"} {"msg":"The guest has requested activation with gen:152","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568343486Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} {"msg":"0fe25c58-4403-46d6-8c3d-d5410bb5af5c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.56835858Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568371364Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"0","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568385488Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"1","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568399492Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"2","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568413286Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.56842745Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568441634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"ds_connection connected from Ok(127.0.0.1:35378)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568466813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"2","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"ds_connection connected from Ok(127.0.0.1:51673)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568516621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"1","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"ds_connection connected from Ok(127.0.0.1:37145)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.568545029Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"io task","client":"0","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569196007Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"1","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569244076Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"2","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569271084Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"0","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569559999Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.56960482Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[0]R flush_numbers[0..12]: [290, 290, 277, 278, 293, 298, 290, 288, 273, 286, 297, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569662255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[0]R generation[0..12]: [148, 148, 141, 142, 149, 151, 148, 148, 138, 147, 150, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569679407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.56969575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[1]R flush_numbers[0..12]: [290, 290, 277, 278, 293, 298, 290, 288, 273, 286, 297, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569711593Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[1]R generation[0..12]: [148, 148, 141, 142, 149, 151, 148, 148, 138, 147, 150, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569727277Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569743749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[2]R flush_numbers[0..12]: [290, 290, 277, 278, 293, 298, 290, 288, 273, 286, 297, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569759623Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[2]R generation[0..12]: [148, 148, 141, 142, 149, 151, 148, 148, 138, 147, 150, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569775026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569791229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Max found gen is 152","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569804633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Generation requested: 152 >= found:152","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:54:27.569818867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Next flush: 299","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569838168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569852862Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"":"downstairs","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569869115Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.56988177Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"0","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569895624Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"1","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569909667Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"client":"2","":"downstairs","session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:54:27.569924251Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569939475Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"0fe25c58-4403-46d6-8c3d-d5410bb5af5c is now active with session: a80b9516-1ed6-4fba-9481-a59374580809","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569954358Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569966843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808,"session_id":"a80b9516-1ed6-4fba-9481-a59374580809"} {"msg":"The guest has finished waiting for activation with:152","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569980057Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} {"msg":"Activated sub_volume 0fe25c58-4403-46d6-8c3d-d5410bb5af5c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:27.569991522Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 524 len 16384 data: 8 11 9 10 02/30 Flush 03/30 Flush 04/30 Write block 19 len 8192 data: 13 12 05/30 Write block 337 len 32768 data: 15 17 14 10 12 15 15 15 06/30 Read block 426 len 24576 07/30 Read block 125 len 40960 08/30 Read block 460 len 24576 09/30 Write block 108 len 8192 data: 11 11 10/30 Write block 441 len 36864 data: 12 10 11 9 8 8 7 8 7 11/30 Read block 378 len 12288 12/30 Read block 528 len 20480 13/30 Read block 195 len 12288 14/30 Write block 211 len 24576 data: 8 9 9 9 8 6 15/30 Write block 209 len 8192 data: 10 8 16/30 Read block 249 len 20480 17/30 Read block 569 len 20480 18/30 Flush 19/30 Write block 387 len 24576 data: 8 9 9 9 10 14 20/30 Write block 170 len 12288 data: 15 15 13 21/30 Read block 534 len 40960 22/30 Read block 90 len 28672 23/30 Write block 528 len 36864 data: 10 13 11 13 14 12 11 9 9 24/30 Write block 259 len 32768 data: 15 14 15 16 12 13 14 17 25/30 Flush 26/30 Flush 27/30 Write block 418 len 16384 data: 11 13 12 13 28/30 Write block 20 len 40960 data: 13 15 15 16 14 11 12 10 13 17 29/30 Write block 525 len 28672 data: 12 10 11 11 14 12 14 30/30 Write block 135 len 32768 data: 13 16 13 13 13 13 14 13 ---------------------------------------------------------------- Crucible gen:152 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 2 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 6 Done Done Done false 1006 Acked Read 10 Done Done Sent false 1007 Acked Read 6 Done Done Sent false 1008 Acked Write 2 Done Done Sent false 1009 Acked Write 9 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 5 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Write 2 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 6 Done Done Sent false 1019 Acked Write 3 Done Done Sent false 1020 Acked Read 10 Done Done Sent false 1021 Acked Read 7 Done Done Sent false 1022 Acked Write 9 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Write 10 Done Done Sent false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 2 24 28 Done 25 25 3 53 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:27.581969191Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1808} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 148 301 301 290 F F T <--- 1 020-039 152 152 148 301 301 290 T T T <--- 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 149 149 149 293 293 293 F F F 5 100-119 152 152 151 301 301 298 F F F <--- 6 120-139 148 148 148 290 290 290 T T F <--- 7 140-159 148 148 148 288 288 288 T T F <--- 8 160-179 152 152 138 302 302 273 F F F <--- 9 180-199 147 147 147 286 286 286 F F F 10 200-219 152 152 150 301 301 297 F F F <--- 11 220-239 151 151 151 298 298 298 F F F 12 240-259 152 152 150 302 302 295 F F F <--- 13 260-279 152 152 150 302 302 296 F F F <--- 14 280-299 150 150 150 294 294 294 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 152 152 151 301 301 298 F F T <--- 17 340-359 152 152 150 301 301 295 F F T <--- 18 360-379 149 149 149 293 293 293 F F F 19 380-399 152 152 151 302 302 298 F F F <--- 20 400-419 138 138 138 274 274 274 T T F <--- 21 420-439 151 151 151 298 298 298 T T F <--- 22 440-459 152 152 150 301 301 296 F F F <--- 23 460-479 148 148 148 289 289 289 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 152 152 152 302 302 299 T T F <--- 27 540-559 147 147 147 286 286 286 F F F 28 560-579 151 151 151 298 298 298 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 152, Max flush: 302 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 76 Verifying data now No change after verify Loop: 76 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 301 301 301 F F F 1 020-039 153 153 153 303 303 303 F F F 2 040-059 141 141 141 277 277 277 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 149 149 149 293 293 293 F F F 5 100-119 152 152 152 301 301 301 F F F 6 120-139 153 153 153 303 303 303 F F F 7 140-159 153 153 153 303 303 303 F F F 8 160-179 152 152 152 302 302 302 F F F 9 180-199 147 147 147 286 286 286 F F F 10 200-219 152 152 152 301 301 301 F F F 11 220-239 151 151 151 298 298 298 F F F 12 240-259 152 152 152 302 302 302 F F F 13 260-279 152 152 152 302 302 302 F F F 14 280-299 150 150 150 294 294 294 F F F 15 300-319 144 144 144 283 283 283 F F F 16 320-339 152 152 152 301 301 301 F F F 17 340-359 152 152 152 301 301 301 F F F 18 360-379 149 149 149 293 293 293 F F F 19 380-399 152 152 152 302 302 302 F F F 20 400-419 153 153 153 303 303 303 F F F 21 420-439 153 153 153 303 303 303 F F F 22 440-459 152 152 152 301 301 301 F F F 23 460-479 148 148 148 289 289 289 F F F 24 480-499 145 145 145 284 284 284 F F F 25 500-519 144 144 144 283 283 283 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 151 151 151 298 298 298 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 153, Max flush: 303 Begin loop 77 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.914751824Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.926539136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.926857768Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.926894482Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.926906896Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.926925039Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} {"msg":"Crucible e533c098-48a2-4db6-a355-8bf1335e5533 has session id: 04f0c517-b13c-4fda-9f8a-1b04b7cbbbca","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927270559Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Upstairs opts: Upstairs UUID: e533c098-48a2-4db6-a355-8bf1335e5533, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927321247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Crucible stats registered with UUID: e533c098-48a2-4db6-a355-8bf1335e5533","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.92733727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:31.927350654Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.92754405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927570969Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927585602Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927708689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"task":"crutest"} {"msg":"The guest has requested activation with gen:154","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927725861Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} {"msg":"e533c098-48a2-4db6-a355-8bf1335e5533 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927741815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927754519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"0","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927768313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"1","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927782137Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"2","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927818351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927833445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927847589Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"ds_connection connected from Ok(127.0.0.1:51583)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927864721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"2","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"ds_connection connected from Ok(127.0.0.1:65507)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927879645Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"0","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"ds_connection connected from Ok(127.0.0.1:47438)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.927894438Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"io task","client":"1","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.928610838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"1","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.928647252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"0","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.928663795Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"2","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.92902061Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929058484Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[0]R flush_numbers[0..12]: [301, 303, 277, 278, 293, 301, 303, 303, 302, 286, 301, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929149834Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[0]R generation[0..12]: [152, 153, 141, 142, 149, 152, 153, 153, 152, 147, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929167587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929184229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[1]R flush_numbers[0..12]: [301, 303, 277, 278, 293, 301, 303, 303, 302, 286, 301, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929200162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[1]R generation[0..12]: [152, 153, 141, 142, 149, 152, 153, 153, 152, 147, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929216026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929232329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[2]R flush_numbers[0..12]: [301, 303, 277, 278, 293, 301, 303, 303, 302, 286, 301, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929250631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[2]R generation[0..12]: [152, 153, 141, 142, 149, 152, 153, 153, 152, 147, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929266094Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929282487Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {Wait for a query_work_queue command to finish before sending IO "msg":"Max found gen is 154","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929302288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Generation requested: 154 >= found:154","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929316822Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Next flush: 304","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929330166Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.92934348Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"":"downstairs","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929357584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929370259Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"0","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929384323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"1","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929398287Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"client":"2","":"downstairs","session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929416099Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929429493Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"e533c098-48a2-4db6-a355-8bf1335e5533 is now active with session: 04f0c517-b13c-4fda-9f8a-1b04b7cbbbca","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929444327Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929457051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816,"session_id":"04f0c517-b13c-4fda-9f8a-1b04b7cbbbca"} {"msg":"The guest has finished waiting for activation with:154","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.929472265Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} {"msg":"Activated sub_volume e533c098-48a2-4db6-a355-8bf1335e5533","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:31.92948377Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 450 len 4096 data: 10 02/30 Write block 21 len 12288 data: 16 16 17 03/30 Read block 376 len 32768 04/30 Write block 253 len 8192 data: 12 10 05/30 Flush 06/30 Write block 470 len 4096 data: 13 07/30 Write block 37 len 36864 data: 10 10 10 11 10 12 11 13 13 08/30 Read block 119 len 36864 09/30 Read block 284 len 24576 10/30 Read block 402 len 32768 11/30 Write block 90 len 28672 data: 9 9 14 15 14 16 14 12/30 Write block 151 len 32768 data: 13 10 9 12 14 13 12 13 13/30 Read block 391 len 4096 14/30 Write block 353 len 24576 data: 15 14 15 18 18 18 15/30 Write block 461 len 36864 data: 11 9 12 14 12 11 13 16 15 16/30 Write block 317 len 28672 data: 10 10 14 14 15 13 12 17/30 Read block 121 len 20480 18/30 Flush 19/30 Write block 499 len 32768 data: 19 14 15 16 14 14 15 17 20/30 Read block 524 len 12288 21/30 Write block 106 len 8192 data: 14 9 22/30 Write block 174 len 36864 data: 9 10 10 9 7 10 11 13 14 23/30 Read block 190 len 40960 24/30 Read block 560 len 20480 25/30 Write block 373 len 36864 data: 8 9 11 12 11 13 12 16 15 26/30 Write block 130 len 4096 data: 9 27/30 Read block 454 len 40960 28/30 Read block 85 len 20480 29/30 Write block 272 len 16384 data: 16 16 14 11 30/30 Read block 555 len 36864 ---------------------------------------------------------------- Crucible gen:154 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 3 Done Done Done false 1002 Acked Read 8 Done Done Done false 1003 Acked Write 2 Done Done Sent false 1004 Acked Flush 0 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 9 Done Done Sent false 1007 Acked Read 9 Done Done Sent false 1008 Acked Read 6 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 8 Done Done Sent false 1012 Acked Read 1 Done Done Sent false 1013 Acked Write 6 Done Done Sent false 1014 Acked Write 9 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 5 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 8 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Write 9 Done Done Sent false 1022 Acked Read 10 Done Done Sent false 1023 Acked Read 5 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Read 10 Done Done Sent false 1027 Acked Read 5 Done Done Sent false 1028 Acked Write 4 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 27 28 Done 29 30 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:31.942048804Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1816} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 301 301 301 F F F 1 020-039 154 154 153 305 305 303 F F T <--- 2 040-059 154 154 141 305 305 277 F F F <--- 3 060-079 142 142 142 278 278 278 F F F 4 080-099 154 154 149 305 305 293 F F F <--- 5 100-119 152 152 152 301 301 301 T T F <--- 6 120-139 153 153 153 303 303 303 T T F <--- 7 140-159 154 154 153 305 305 303 F F F <--- 8 160-179 152 152 152 302 302 302 T T F <--- 9 180-199 147 147 147 286 286 286 T T F <--- 10 200-219 152 152 152 301 301 301 F F F 11 220-239 151 151 151 298 298 298 F F F 12 240-259 154 154 152 304 304 302 F F F <--- 13 260-279 152 152 152 302 302 302 T T F <--- 14 280-299 150 150 150 294 294 294 F F F 15 300-319 154 154 144 305 305 283 F F F <--- 16 320-339 154 154 152 305 305 301 F F F <--- 17 340-359 154 154 152 305 305 301 F F F <--- 18 360-379 149 149 149 293 293 293 T T F <--- 19 380-399 152 152 152 302 302 302 T T F <--- 20 400-419 153 153 153 303 303 303 F F F 21 420-439 153 153 153 303 303 303 F F F 22 440-459 154 154 152 304 304 301 F F T <--- 23 460-479 154 154 148 305 305 289 F F F <--- 24 480-499 145 145 145 284 284 284 T T F <--- 25 500-519 144 144 144 283 283 283 T T F <--- 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 151 151 151 298 298 298 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 154, Max flush: 305 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 77 Verifying data now No change after verify Loop: 77 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 301 301 301 F F F 1 020-039 154 154 154 305 305 305 F F F 2 040-059 154 154 154 305 305 305 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 154 154 154 305 305 305 F F F 5 100-119 155 155 155 306 306 306 F F F 6 120-139 155 155 155 306 306 306 F F F 7 140-159 154 154 154 305 305 305 F F F 8 160-179 155 155 155 306 306 306 F F F 9 180-199 155 155 155 306 306 306 F F F 10 200-219 152 152 152 301 301 301 F F F 11 220-239 151 151 151 298 298 298 F F F 12 240-259 154 154 154 304 304 304 F F F 13 260-279 155 155 155 306 306 306 F F F 14 280-299 150 150 150 294 294 294 F F F 15 300-319 154 154 154 305 305 305 F F F 16 320-339 154 154 154 305 305 305 F F F 17 340-359 154 154 154 305 305 305 F F F 18 360-379 155 155 155 306 306 306 F F F 19 380-399 155 155 155 306 306 306 F F F 20 400-419 153 153 153 303 303 303 F F F 21 420-439 153 153 153 303 303 303 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 155 155 155 306 306 306 F F F 25 500-519 155 155 155 306 306 306 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 151 151 151 298 298 298 F F F 29 580-599 147 147 147 286 286 286 F F F Max gen: 155, Max flush: 306 Begin loop 78 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.285011027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.296796969Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.29714052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297178584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297190648Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297208301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} {"msg":"Crucible a38cfcc9-8d62-4183-bfd7-c20c393c1563 has session id: 6546de73-1e1e-41c7-b24f-c9e2d218fa48","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297552871Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Upstairs opts: Upstairs UUID: a38cfcc9-8d62-4183-bfd7-c20c393c1563, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297615004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Crucible stats registered with UUID: a38cfcc9-8d62-4183-bfd7-c20c393c1563","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297631058Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:36.297643892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297842346Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297885637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297904059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.297988842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"task":"crutest"} {"msg":"The guest has requested activation with gen:156","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298013172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} {"msg":"a38cfcc9-8d62-4183-bfd7-c20c393c1563 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298024797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298037731Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"0","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298051905Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"1","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298065709Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"2","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298079353Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298093507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298107561Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"ds_connection connected from Ok(127.0.0.1:58562)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298122505Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"2","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"ds_connection connected from Ok(127.0.0.1:41632)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298137198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"0","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"ds_connection connected from Ok(127.0.0.1:34788)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298151932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"io task","client":"1","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298838714Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"0","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298867242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"2","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.298884304Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"1","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299206855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299258372Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[0]R flush_numbers[0..12]: [301, 305, 305, 278, 305, 306, 306, 305, 306, 306, 301, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299324544Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[0]R generation[0..12]: [152, 154, 154, 142, 154, 155, 155, 154, 155, 155, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299342466Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299359209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[1]R flush_numbers[0..12]: [301, 305, 305, 278, 305, 306, 306, 305, 306, 306, 301, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299374992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[1]R generation[0..12]: [152, 154, 154, 142, 154, 155, 155, 154, 155, 155, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299390835Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299407088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[2]R flush_numbers[0..12]: [301, 305, 305, 278, 305, 306, 306, 305, 306, 306, 301, 298]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299422691Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[2]R generation[0..12]: [152, 154, 154, 142, 154, 155, 155, 154, 155, 155, 152, 151]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299438274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299454457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Max found gen is 156","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299467672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Generation requested: 156 >= found:156","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299481805Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Next flush: 307","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.29949517Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299511463Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"":"downstairs","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible"Wait for a query_work_queue command to finish before sending IO ,"level":30,"time":"2026-03-12T22:54:36.299532214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299544868Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"0","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299559162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"1","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299572906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"client":"2","":"downstairs","session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.29958751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299601024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"a38cfcc9-8d62-4183-bfd7-c20c393c1563 is now active with session: 6546de73-1e1e-41c7-b24f-c9e2d218fa48","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299615787Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299631101Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824,"session_id":"6546de73-1e1e-41c7-b24f-c9e2d218fa48"} {"msg":"The guest has finished waiting for activation with:156","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.299644465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} {"msg":"Activated sub_volume a38cfcc9-8d62-4183-bfd7-c20c393c1563","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:36.29965585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 355 len 36864 data: 16 19 19 19 17 15 16 11 12 02/30 Read block 276 len 32768 03/30 Write block 572 len 20480 data: 14 12 12 10 8 04/30 Read block 432 len 8192 05/30 Flush 06/30 Write block 571 len 20480 data: 8 15 13 13 11 07/30 Read block 353 len 24576 08/30 Read block 241 len 24576 09/30 Read block 260 len 20480 10/30 Read block 77 len 8192 11/30 Flush 12/30 Write block 142 len 36864 data: 14 10 12 12 11 13 13 13 12 13/30 Read block 413 len 24576 14/30 Read block 219 len 24576 15/30 Read block 153 len 24576 16/30 Read block 18 len 32768 17/30 Read block 11 len 24576 18/30 Write block 380 len 20480 data: 17 16 16 13 11 19/30 Write block 127 len 12288 data: 12 10 10 20/30 Read block 113 len 32768 21/30 Write block 169 len 28672 data: 15 16 16 14 9 10 11 22/30 Flush 23/30 Write block 170 len 32768 data: 17 17 15 10 11 12 11 10 24/30 Write block 596 len 16384 data: 9 8 6 5 25/30 Read block 126 len 12288 26/30 Write block 235 len 20480 data: 15 16 16 16 15 27/30 Read block 385 len 16384 28/30 Read block 548 len 12288 29/30 Write block 328 len 8192 data: 16 12 30/30 Read block 553 len 16384 ---------------------------------------------------------------- Crucible gen:156 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Sent Done false 1001 Acked Read 8 Done Sent Done false 1002 Acked Write 5 Done Sent Done false 1003 Acked Read 2 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 5 Done Sent Done false 1006 Acked Read 6 Done Sent Done false 1007 Acked Read 6 Done Sent Done false 1008 Acked Read 5 Done Sent Done false 1009 Acked Read 2 Done Sent Done false 1010 Acked Flush 0 Done Sent Done false 1011 Acked Write 9 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Read 6 Done Sent Done false 1014 Acked Read 6 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 6 Done Sent Done false 1017 Acked Write 5 Done Sent Done false 1018 Acked Write 3 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Write 7 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 8 Done Sent Done false 1023 Acked Write 4 Done Sent Done false 1024 Acked Read 3 Done Sent Done false 1025 Acked Write 5 Done Sent Done false 1026 Acked Read 4 Done Sent Done false 1027 Acked Read 3 Done Sent Done false 1028 Acked Write 2 Done Sent Done false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 30 1 31 Done 30 0 29 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:36.312698567Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1824} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 301 301 301 F F F 1 020-039 154 154 154 305 305 305 F F F 2 040-059 154 154 154 305 305 305 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 154 154 154 305 305 305 F F F 5 100-119 155 155 155 306 306 306 F F F 6 120-139 156 155 156 309 306 309 F F F <--- 7 140-159 156 154 156 309 305 309 F F F <--- 8 160-179 156 155 156 309 306 309 T F T <--- 9 180-199 155 155 155 306 306 306 F F F 10 200-219 152 152 152 301 301 301 F F F 11 220-239 151 151 151 298 298 298 T F T <--- 12 240-259 154 154 154 304 304 304 F F F 13 260-279 155 155 155 306 306 306 F F F 14 280-299 150 150 150 294 294 294 F F F 15 300-319 154 154 154 305 305 305 F F F 16 320-339 154 154 154 305 305 305 T F T <--- 17 340-359 156 154 156 307 305 307 F F F <--- 18 360-379 156 155 156 307 306 307 F F F <--- 19 380-399 156 155 156 309 306 309 F F F <--- 20 400-419 153 153 153 303 303 303 F F F 21 420-439 153 153 153 303 303 303 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 155 155 155 306 306 306 F F F 25 500-519 155 155 155 306 306 306 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 156 151 156 308 298 308 F F F <--- 29 580-599 147 147 147 286 286 286 T F T <--- Max gen: 156, Max flush: 309 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 78 Verifying data now No change after verify Loop: 78 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 152 152 301 301 301 F F F 1 020-039 154 154 154 305 305 305 F F F 2 040-059 154 154 154 305 305 305 F F F 3 060-079 142 142 142 278 278 278 F F F 4 080-099 154 154 154 305 305 305 F F F 5 100-119 155 155 155 306 306 306 F F F 6 120-139 156 156 156 309 309 309 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 157 157 157 310 310 310 F F F 9 180-199 155 155 155 306 306 306 F F F 10 200-219 152 152 152 301 301 301 F F F 11 220-239 157 157 157 310 310 310 F F F 12 240-259 154 154 154 304 304 304 F F F 13 260-279 155 155 155 306 306 306 F F F 14 280-299 150 150 150 294 294 294 F F F 15 300-319 154 154 154 305 305 305 F F F 16 320-339 157 157 157 310 310 310 F F F 17 340-359 156 156 156 307 307 307 F F F 18 360-379 156 156 156 307 307 307 F F F 19 380-399 156 156 156 309 309 309 F F F 20 400-419 153 153 153 303 303 303 F F F 21 420-439 153 153 153 303 303 303 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 155 155 155 306 306 306 F F F 25 500-519 155 155 155 306 306 306 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 156 156 156 308 308 308 F F F 29 580-599 157 157 157 310 310 310 F F F Max gen: 157, Max flush: 310 Begin loop 79 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.627341034Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:40.638103269Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.63814744Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.648562906Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.648862626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.648903849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.648919142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.648937024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} {"msg":"Crucible 34d9b2af-5682-42df-9feb-46ce0893f03c has session id: c3cc6530-282a-4a38-b757-3c48d4bbad36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649285043Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Upstairs opts: Upstairs UUID: 34d9b2af-5682-42df-9feb-46ce0893f03c, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649340219Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Crucible stats registered with UUID: 34d9b2af-5682-42df-9feb-46ce0893f03c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649356402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:40.649369597Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649542902Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649567931Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649582504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.64970791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"task":"crutest"} {"msg":"The guest has requested activation with gen:158","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649724883Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} {"msg":"34d9b2af-5682-42df-9feb-46ce0893f03c active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649741855Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.64975471Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"0","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649768474Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"1","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649784627Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"2","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649798291Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649812515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649826629Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"ds_connection connected from Ok(127.0.0.1:53328)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649841392Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"2","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"ds_connection connected from Ok(127.0.0.1:40768)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.649855816Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"1","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"ds_connection connected from Ok(127.0.0.1:40345)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.64987071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"io task","client":"0","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.650602312Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"1","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.65063044Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"0","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.650647533Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"2","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.650970413Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651015014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[0]R flush_numbers[0..12]: [301, 305, 305, 278, 305, 306, 309, 309, 310, 306, 301, 310]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651085493Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[0]R generation[0..12]: [152, 154, 154, 142, 154, 155, 156, 156, 157, 155, 152, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651102736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651119318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[1]R flush_numbers[0..12]: [301, 305, 305, 278, 305, 306, 309, 309, 310, 306, 301, 310]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.65113796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[1]R generation[0..12]: [152, 154, 154, 142, 154, 155, 156, 156, 157, 155, 152, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651154003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651170306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[2]R flush_numbers[0..12]: [301, 305, 305, 278, 305, 306, 309, 309, 310, 306, 301, 310]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651186199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[2]R generation[0..12]: [152, 154, 154, 142, 154, 155, 156, 156, 157, 155, 152, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651201763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651218385Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Max found gen is 158","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:54:40.65123179Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Generation requested: 158 >= found:158","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651249922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Next flush: 311","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651262966Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.65127633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"":"downstairs","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651290464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651303229Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"0","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651316863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"1","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Transition from WaitQuorum to Active","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651333805Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"client":"2","":"downstairs","session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651350218Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651363663Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"34d9b2af-5682-42df-9feb-46ce0893f03c is now active with session: c3cc6530-282a-4a38-b757-3c48d4bbad36","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651378336Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651390891Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832,"session_id":"c3cc6530-282a-4a38-b757-3c48d4bbad36"} {"msg":"The guest has finished waiting for activation with:158","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.651404095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} {"msg":"Activated sub_volume 34d9b2af-5682-42df-9feb-46ce0893f03c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:40.65141545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 331 len 36864 data: 13 14 12 10 13 16 16 18 15 02/30 Write block 104 len 40960 data: 16 13 15 10 12 12 12 11 9 9 03/30 Read block 556 len 8192 04/30 Read block 300 len 4096 05/30 Read block 67 len 28672 06/30 Write block 377 len 4096 data: 12 07/30 Write block 347 len 24576 data: 12 10 15 17 17 16 08/30 Read block 411 len 24576 09/30 Write block 573 len 40960 data: 14 14 12 9 7 10 12 11 11 10 10/30 Write block 390 len 12288 data: 10 11 15 11/30 Write block 588 len 4096 data: 13 12/30 Read block 347 len 24576 13/30 Read block 46 len 8192 14/30 Read block 216 len 40960 15/30 Write block 306 len 36864 data: 7 7 8 10 8 10 10 6 6 16/30 Read block 4 len 36864 17/30 Write block 572 len 20480 data: 16 15 15 13 10 18/30 Read block 427 len 40960 19/30 Read block 473 len 8192 20/30 Write block 361 len 36864 data: 17 12 13 11 8 10 10 11 10 21/30 Flush 22/30 Write block 290 len 20480 data: 17 17 15 14 13 23/30 Write block 13 len 8192 data: 13 15 24/30 Flush 25/30 Write block 199 len 28672 data: 18 17 16 19 21 21 19 26/30 Read block 560 len 12288 27/30 Read block 293 len 24576 28/30 Write block 421 len 36864 data: 14 13 13 13 15 12 9 11 14 29/30 Write block 60 len 36864 data: 9 7 8 9 10 10 10 9 8 30/30 Write block 398 len 24576 data: 7 6 7 7 7 9 ---------------------------------------------------------------- Crucible gen:158 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 9 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 1 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Write 6 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Write 10 Sent Done Done false 1009 Acked Write 3 Sent Done Done false 1010 Acked Write 1 Sent Done Done false 1011 Acked Read 6 Sent Done Done false 1012 Acked Read 2 Sent Done Done false 1013 Acked Read 10 Sent Done Done false 1014 Acked Write 9 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Write 5 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Read 2 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Flush 0 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Flush 0 Sent Done Done false 1024 Acked Write 7 Sent Done Done false 1025 Acked Read 3 Sent Done Done false 1026 Acked Read 6 Sent Done Done false 1027 Acked Write 9 Sent Done Done false 1028 Acked Write 9 Sent Sent Sent false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 2 2 31 Done 3 28 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1023)) Some(JobId(1023)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:40.664368316Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1832} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 152 158 158 301 312 312 F F F <--- 1 020-039 154 154 154 305 305 305 F F F 2 040-059 154 154 154 305 305 305 F F F 3 060-079 142 142 142 278 278 278 F T T <--- 4 080-099 154 154 154 305 305 305 F F F 5 100-119 155 158 158 306 311 311 T F F <--- 6 120-139 156 156 156 309 309 309 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 157 157 157 310 310 310 F F F 9 180-199 155 155 155 306 306 306 F T T <--- 10 200-219 152 152 152 301 301 301 F T T <--- 11 220-239 157 157 157 310 310 310 F F F 12 240-259 154 154 154 304 304 304 F F F 13 260-279 155 155 155 306 306 306 F F F 14 280-299 150 158 158 294 312 312 F F F <--- 15 300-319 154 158 158 305 311 311 F F F <--- 16 320-339 157 158 158 310 311 311 T F F <--- 17 340-359 156 158 158 307 311 311 F F F <--- 18 360-379 156 158 158 307 311 311 F F F <--- 19 380-399 156 158 158 309 311 311 F T T <--- 20 400-419 153 153 153 303 303 303 F T T <--- 21 420-439 153 153 153 303 303 303 F T T <--- 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 155 155 155 306 306 306 F F F 25 500-519 155 155 155 306 306 306 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 156 158 158 308 311 311 F F F <--- 29 580-599 157 158 158 310 311 311 F F F <--- Max gen: 158, Max flush: 312 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 79 Verifying data now No change after verify Loop: 79 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 312 312 312 F F F 1 020-039 154 154 154 305 305 305 F F F 2 040-059 154 154 154 305 305 305 F F F 3 060-079 159 159 159 313 313 313 F F F 4 080-099 154 154 154 305 305 305 F F F 5 100-119 158 158 158 311 311 311 F F F 6 120-139 156 156 156 309 309 309 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 157 157 157 310 310 310 F F F 9 180-199 159 159 159 313 313 313 F F F 10 200-219 159 159 159 313 313 313 F F F 11 220-239 157 157 157 310 310 310 F F F 12 240-259 154 154 154 304 304 304 F F F 13 260-279 155 155 155 306 306 306 F F F 14 280-299 158 158 158 312 312 312 F F F 15 300-319 158 158 158 311 311 311 F F F 16 320-339 158 158 158 311 311 311 F F F 17 340-359 158 158 158 311 311 311 F F F 18 360-379 158 158 158 311 311 311 F F F 19 380-399 159 159 159 313 313 313 F F F 20 400-419 159 159 159 313 313 313 F F F 21 420-439 159 159 159 313 313 313 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 155 155 155 306 306 306 F F F 25 500-519 155 155 155 306 306 306 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 158 158 158 311 311 311 F F F 29 580-599 158 158 158 311 311 311 F F F Max gen: 159, Max flush: 313 Begin loop 80 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.004788942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.016717912Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01704526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017082983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017095708Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01711354Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} {"msg":"Crucible e12bed74-ee1f-46fa-b307-24ced3093da4 has session id: 72cef4eb-bf8e-4fe9-a07e-b87341e93867","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017448395Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Upstairs opts: Upstairs UUID: e12bed74-ee1f-46fa-b307-24ced3093da4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01748292Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Crucible stats registered with UUID: e12bed74-ee1f-46fa-b307-24ced3093da4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017501462Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:45.017515246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017689631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017726465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017741998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017858097Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"task":"crutest"} {"msg":"The guest has requested activation with gen:160","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01787518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} {"msg":"e12bed74-ee1f-46fa-b307-24ced3093da4 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017890374Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017903278Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"0","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017917252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"1","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017931216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"2","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01794506Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017959364Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017973537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"ds_connection connected from Ok(127.0.0.1:57801)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.017988751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"0","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"ds_connection connected from Ok(127.0.0.1:47844)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.018119374Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"2","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"ds_connection connected from Ok(127.0.0.1:62882)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.018169193Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"io task","client":"1","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.018789004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"2","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.018829007Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"1","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.018846169Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"0","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019175227Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019221846Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[0]R flush_numbers[0..12]: [312, 305, 305, 313, 305, 311, 309, 309, 310, 313, 313, 310]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019277832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[0]R generation[0..12]: [158, 154, 154, 159, 154, 158, 156, 156, 157, 159, 159, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019295165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019311817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[1]R flush_numbers[0..12]: [312, 305, 305, 313, 305, 311, 309, 309, 310, 313, 313, 310]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019327721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[1]R generation[0..12]: [158, 154, 154, 159, 154, 158, 156, 156, 157, 159, 159, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019343314Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019359917Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[2]R flush_numbers[0..12]: [312, 305, 305, 313, 305, 311, 309, 309, 310, 313, 313, 310]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01937557Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[2]R generation[0..12]: [158, 154, 154, 159, 154, 158, 156, 156, 157, 159, 159, 157]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019391433Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019410705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 160","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019430226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Generation requested: 160 >= found:160","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01944465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Next flush: 314","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019458014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019471378Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"":"downstairs","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019485872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019498666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"0","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.01951308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"1","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019528803Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"client":"2","":"downstairs","session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019543527Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019556981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"e12bed74-ee1f-46fa-b307-24ced3093da4 is now active with session: 72cef4eb-bf8e-4fe9-a07e-b87341e93867","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019571875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019584779Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840,"session_id":"72cef4eb-bf8e-4fe9-a07e-b87341e93867"} {"msg":"The guest has finished waiting for activation with:160","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019598283Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} {"msg":"Activated sub_volume e12bed74-ee1f-46fa-b307-24ced3093da4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:45.019610308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 42 len 8192 data: 13 12 02/30 Read block 45 len 40960 03/30 Read block 213 len 12288 04/30 Write block 385 len 16384 data: 10 10 9 10 05/30 Write block 362 len 8192 data: 13 14 06/30 Read block 308 len 8192 07/30 Write block 229 len 40960 data: 9 9 9 12 14 16 16 17 17 17 08/30 Read block 496 len 28672 09/30 Flush 10/30 Write block 491 len 32768 data: 15 14 13 15 17 18 17 16 11/30 Write block 573 len 4096 data: 16 12/30 Write block 162 len 24576 data: 17 13 10 8 9 11 13/30 Write block 20 len 12288 data: 14 17 17 14/30 Write block 221 len 28672 data: 8 10 15 13 8 7 8 15/30 Write block 255 len 16384 data: 12 11 14 14 16/30 Read block 461 len 32768 17/30 Write block 94 len 16384 data: 15 17 15 14 18/30 Read block 276 len 20480 19/30 Read block 329 len 28672 20/30 Read block 124 len 32768 21/30 Read block 551 len 24576 22/30 Flush 23/30 Write block 494 len 28672 data: 16 18 19 18 17 20 15 24/30 Read block 29 len 4096 25/30 Write block 19 len 16384 data: 14 15 18 18 26/30 Write block 421 len 28672 data: 15 14 14 14 16 13 10 27/30 Write block 110 len 16384 data: 13 12 10 10 28/30 Write block 341 len 20480 data: 13 16 16 16 15 29/30 Read block 86 len 32768 30/30 Write block 285 len 20480 data: 19 21 21 18 17 ---------------------------------------------------------------- Crucible gen:160 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Sent Done false 1001 Acked Read 10 Done Sent Done false 1002 Acked Read 3 Done Sent Done false 1003 Acked Write 4 Done Sent Done false 1004 Acked Write 2 Done Sent Done false 1005 Acked Read 2 Done Sent Done false 1006 Acked Write 10 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Flush 0 Done Sent Done false 1009 Acked Write 8 Done Sent Done false 1010 Acked Write 1 Done Sent Done false 1011 Acked Write 6 Done Sent Done false 1012 Acked Write 3 Done Sent Done false 1013 Acked Write 7 Done Sent Done false 1014 Acked Write 4 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Write 4 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 8 Done Sent Done false 1020 Acked Read 6 Done Sent Done false 1021 Acked Flush 0 Done Sent Done false 1022 Acked Write 7 Done Sent Done false 1023 Acked Read 1 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Write 7 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 8 Done Sent Done false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1021)) None Some(JobId(1021)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:45.031286258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1840} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 158 158 158 312 312 312 T F T <--- 1 020-039 160 154 160 315 305 315 T F T <--- 2 040-059 160 154 160 314 305 314 F F F <--- 3 060-079 159 159 159 313 313 313 F F F 4 080-099 160 154 160 315 305 315 F F F <--- 5 100-119 158 158 158 311 311 311 T F T <--- 6 120-139 156 156 156 309 309 309 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 157 160 315 310 315 F F F <--- 9 180-199 159 159 159 313 313 313 F F F 10 200-219 159 159 159 313 313 313 F F F 11 220-239 160 157 160 315 310 315 F F F <--- 12 240-259 160 154 160 315 304 315 F F F <--- 13 260-279 155 155 155 306 306 306 F F F 14 280-299 158 158 158 312 312 312 T F T <--- 15 300-319 158 158 158 311 311 311 F F F 16 320-339 158 158 158 311 311 311 F F F 17 340-359 158 158 158 311 311 311 T F T <--- 18 360-379 160 158 160 314 311 314 F F F <--- 19 380-399 160 159 160 314 313 314 F F F <--- 20 400-419 159 159 159 313 313 313 F F F 21 420-439 159 159 159 313 313 313 T F T <--- 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 160 155 160 315 306 315 T F T <--- 25 500-519 155 155 155 306 306 306 T F T <--- 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 160 158 160 315 311 315 F F F <--- 29 580-599 158 158 158 311 311 311 F F F Max gen: 160, Max flush: 315 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 80 Verifying data now No change after verify Loop: 80 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 161 161 161 316 316 316 F F F 1 020-039 161 161 161 316 316 316 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 159 159 159 313 313 313 F F F 4 080-099 160 160 160 315 315 315 F F F 5 100-119 161 161 161 316 316 316 F F F 6 120-139 156 156 156 309 309 309 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 159 159 159 313 313 313 F F F 10 200-219 159 159 159 313 313 313 F F F 11 220-239 160 160 160 315 315 315 F F F 12 240-259 160 160 160 315 315 315 F F F 13 260-279 155 155 155 306 306 306 F F F 14 280-299 161 161 161 316 316 316 F F F 15 300-319 158 158 158 311 311 311 F F F 16 320-339 158 158 158 311 311 311 F F F 17 340-359 161 161 161 316 316 316 F F F 18 360-379 160 160 160 314 314 314 F F F 19 380-399 160 160 160 314 314 314 F F F 20 400-419 159 159 159 313 313 313 F F F 21 420-439 161 161 161 316 316 316 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 154 154 154 305 305 305 F F F 24 480-499 161 161 161 316 316 316 F F F 25 500-519 161 161 161 316 316 316 F F F 26 520-539 153 153 153 303 303 303 F F F 27 540-559 147 147 147 286 286 286 F F F 28 560-579 160 160 160 315 315 315 F F F 29 580-599 158 158 158 311 311 311 F F F Max gen: 161, Max flush: 316 Begin loop 81 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.365991819Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.37766606Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.377984322Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378023435Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.37803554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378053252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} {"msg":"Crucible 5f370bf8-f7a6-4763-8071-34922f5b2e86 has session id: 492bf4a0-2f4e-4ad7-a915-3b9cb13fe916","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378389716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Upstairs opts: Upstairs UUID: 5f370bf8-f7a6-4763-8071-34922f5b2e86, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378424681Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Crucible stats registered with UUID: 5f370bf8-f7a6-4763-8071-34922f5b2e86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378440204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:49.378453509Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378646265Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.37868134Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378696863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378794391Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"task":"crutest"} {"msg":"The guest has requested activation with gen:162","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378811583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} {"msg":"5f370bf8-f7a6-4763-8071-34922f5b2e86 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378827736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.37884286Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"0","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378857024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"1","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378871088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"2","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378884802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378898976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378913439Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"ds_connection connected from Ok(127.0.0.1:59555)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.378982399Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"2","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"ds_connection connected from Ok(127.0.0.1:65387)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.379009528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"0","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"ds_connection connected from Ok(127.0.0.1:48354)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.37902819Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"io task","client":"1","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.37958057Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"1","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.379611787Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"0","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.379920973Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.379959436Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380220673Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"2","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[0]R flush_numbers[0..12]: [316, 316, 314, 313, 315, 316, 309, 309, 315, 313, 313, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380592771Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[0]R generation[0..12]: [161, 161, 160, 159, 160, 161, 156, 156, 160, 159, 159, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380612473Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380629246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[1]R flush_numbers[0..12]: [316, 316, 314, 313, 315, 316, 309, 309, 315, 313, 313, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380645139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[1]R generation[0..12]: [161, 161, 160, 159, 160, 161, 156, 156, 160, 159, 159, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380660602Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380678014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[2]R flush_numbers[0..12]: [316, 316, 314, 313, 315, 316, 309, 309, 315, 313, 313, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380693778Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[2]R generation[0..12]: [161, 161, 160, 159, 160, 161, 156, 156, 160, 159, 159, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380709361Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380725644Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 162","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380744945Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Generation requested: 162 >= found:162","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380759259Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Next flush: 317","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380772434Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380785738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"":"downstairs","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380799422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380812136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"0","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:54:49.380828139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"1","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380845452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"client":"2","":"downstairs","session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380860026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.3808735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"5f370bf8-f7a6-4763-8071-34922f5b2e86 is now active with session: 492bf4a0-2f4e-4ad7-a915-3b9cb13fe916","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380888773Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380901418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848,"session_id":"492bf4a0-2f4e-4ad7-a915-3b9cb13fe916"} {"msg":"The guest has finished waiting for activation with:162","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380914912Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} {"msg":"Activated sub_volume 5f370bf8-f7a6-4763-8071-34922f5b2e86","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:49.380926447Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 80 len 32768 data: 14 12 11 12 13 13 12 11 03/30 Read block 422 len 4096 04/30 Read block 224 len 36864 05/30 Write block 273 len 24576 data: 17 15 12 9 10 9 06/30 Write block 25 len 4096 data: 12 07/30 Write block 465 len 28672 data: 13 12 14 17 16 14 13 08/30 Write block 14 len 40960 data: 16 14 16 16 14 15 16 19 19 18 09/30 Write block 287 len 16384 data: 22 19 18 18 10/30 Read block 432 len 28672 11/30 Read block 112 len 12288 12/30 Read block 435 len 36864 13/30 Read block 212 len 32768 14/30 Write block 199 len 8192 data: 19 18 15/30 Write block 285 len 24576 data: 20 22 23 20 19 19 16/30 Flush 17/30 Write block 399 len 28672 data: 7 8 8 8 10 12 13 18/30 Read block 517 len 12288 19/30 Read block 280 len 20480 20/30 Write block 248 len 40960 data: 10 10 12 12 11 13 11 13 12 15 21/30 Write block 391 len 32768 data: 12 16 13 9 6 6 6 8 22/30 Write block 124 len 24576 data: 15 16 14 13 11 11 23/30 Flush 24/30 Write block 572 len 28672 data: 17 17 16 14 11 8 11 25/30 Write block 543 len 36864 data: 9 9 13 10 13 15 13 14 10 26/30 Read block 533 len 12288 27/30 Write block 254 len 20480 data: 12 14 13 16 15 28/30 Write block 525 len 32768 data: 13 11 12 12 15 13 15 15 29/30 Write block 328 len 36864 data: 17 13 13 14 15 13 11 14 17 30/30 Read block 103 len 36864 ---------------------------------------------------------------- Crucible gen:162 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 8 Done Done Done false 1002 Acked Read 1 Done Done Sent false 1003 Acked Read 9 Done Done Sent false 1004 Acked Write 6 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Write 7 Done Done Sent false 1007 Acked Write 10 Done Done Sent false 1008 Acked Write 4 Done Done Sent false 1009 Acked Read 7 Done Done Sent false 1010 Acked Read 3 Done Done Sent false 1011 Acked Read 9 Done Done Sent false 1012 Acked Read 8 Done Done Sent false 1013 Acked Write 2 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Flush 0 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Read 3 Done Done Sent false 1018 Acked Read 5 Done Done Sent false 1019 Acked Write 10 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 6 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 7 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 5 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 9 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 0 28 29 Done 28 29 1 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) Some(JobId(1000)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:49.393659488Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1848} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 161 318 318 316 F F F <--- 1 020-039 162 162 161 318 318 316 F F F <--- 2 040-059 160 160 160 314 314 314 F F F 3 060-079 159 159 159 313 313 313 F F F 4 080-099 162 162 160 318 318 315 F F T <--- 5 100-119 161 161 161 316 316 316 F F F 6 120-139 162 162 156 319 319 309 F F F <--- 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 162 162 159 318 318 313 F F F <--- 10 200-219 162 162 159 318 318 313 F F F <--- 11 220-239 160 160 160 315 315 315 F F F 12 240-259 162 162 160 319 319 315 T T F <--- 13 260-279 162 162 155 318 318 306 F F F <--- 14 280-299 162 162 161 318 318 316 F F F <--- 15 300-319 158 158 158 311 311 311 F F F 16 320-339 158 158 158 311 311 311 T T F <--- 17 340-359 161 161 161 316 316 316 F F F 18 360-379 160 160 160 314 314 314 F F F 19 380-399 162 162 160 319 319 314 F F F <--- 20 400-419 162 162 159 319 319 313 F F F <--- 21 420-439 161 161 161 316 316 316 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 162 162 154 318 318 305 F F F <--- 24 480-499 161 161 161 316 316 316 F F F 25 500-519 161 161 161 316 316 316 F F F 26 520-539 153 153 153 303 303 303 T T F <--- 27 540-559 147 147 147 286 286 286 T T F <--- 28 560-579 160 160 160 315 315 315 T T F <--- 29 580-599 158 158 158 311 311 311 F F F Max gen: 162, Max flush: 319 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 81 Verifying data now No change after verify Loop: 81 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 159 159 159 313 313 313 F F F 4 080-099 162 162 162 318 318 318 F F F 5 100-119 161 161 161 316 316 316 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 162 162 162 318 318 318 F F F 10 200-219 162 162 162 318 318 318 F F F 11 220-239 160 160 160 315 315 315 F F F 12 240-259 163 163 163 320 320 320 F F F 13 260-279 162 162 162 318 318 318 F F F 14 280-299 162 162 162 318 318 318 F F F 15 300-319 158 158 158 311 311 311 F F F 16 320-339 163 163 163 320 320 320 F F F 17 340-359 161 161 161 316 316 316 F F F 18 360-379 160 160 160 314 314 314 F F F 19 380-399 162 162 162 319 319 319 F F F 20 400-419 162 162 162 319 319 319 F F F 21 420-439 161 161 161 316 316 316 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 162 162 162 318 318 318 F F F 24 480-499 161 161 161 316 316 316 F F F 25 500-519 161 161 161 316 316 316 F F F 26 520-539 163 163 163 320 320 320 F F F 27 540-559 163 163 163 320 320 320 F F F 28 560-579 163 163 163 320 320 320 F F F 29 580-599 158 158 158 311 311 311 F F F Max gen: 163, Max flush: 320 Begin loop 82 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.72187901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.733631047Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.733952398Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73399196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734004165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734022517Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} {"msg":"Crucible 051f18ad-b3bd-40e6-8b14-2cb04de99ea9 has session id: 8f22a963-ee00-45b2-ac37-c4aea86baf3e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734379832Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Upstairs opts: Upstairs UUID: 051f18ad-b3bd-40e6-8b14-2cb04de99ea9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734459188Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Crucible stats registered with UUID: 051f18ad-b3bd-40e6-8b14-2cb04de99ea9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73447665Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:53.734490414Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734681352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734716716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734732489Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734903705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"task":"crutest"} {"msg":"The guest has requested activation with gen:164","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734920728Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} {"msg":"051f18ad-b3bd-40e6-8b14-2cb04de99ea9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734944528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734957382Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"0","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734971456Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"1","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73498526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"2","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.734999114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735013518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735027801Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:48545)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73512395Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"0","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:47505)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735212052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"2","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"ds_connection connected from Ok(127.0.0.1:61437)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735240139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"io task","client":"1","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735882641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"1","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735903022Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"0","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.735919295Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"2","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73625363Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73630122Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 314, 313, 318, 316, 319, 309, 315, 318, 318, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736371789Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[0]R generation[0..12]: [162, 162, 160, 159, 162, 161, 162, 156, 160, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736389012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736405754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 314, 313, 318, 316, 319, 309, 315, 318, 318, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736421727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[1]R generation[0..12]: [162, 162, 160, 159, 162, 161, 162, 156, 160, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736437151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736453374Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 314, 313, 318, 316, 319, 309, 315, 318, 318, 315]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736472305Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[2]R generation[0..12]: [162, 162, 160, 159, 162, 161, 162, 156, 160, 162, 162, 160]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736487979Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736503942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Max found gen is 164","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:54:53.736517766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Generation requested: 164 >= found:164","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736537347Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Next flush: 321","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736550801Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736564246Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"":"downstairs","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.73657811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736590534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"0","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736604518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1856,"client":"1","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736621591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"client":"2","":"downstairs","session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736636354Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736649748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"051f18ad-b3bd-40e6-8b14-2cb04de99ea9 is now active with session: 8f22a963-ee00-45b2-ac37-c4aea86baf3e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736664632Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736677257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856,"session_id":"8f22a963-ee00-45b2-ac37-c4aea86baf3e"} {"msg":"The guest has finished waiting for activation with:164","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736690701Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} {"msg":"Activated sub_volume 051f18ad-b3bd-40e6-8b14-2cb04de99ea9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:53.736703715Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 584 len 12288 data: 10 11 13 02/30 Read block 213 len 36864 03/30 Write block 104 len 24576 data: 17 14 16 11 13 13 04/30 Read block 415 len 40960 05/30 Read block 145 len 16384 06/30 Read block 560 len 8192 07/30 Write block 306 len 32768 data: 8 8 9 11 9 11 11 7 08/30 Read block 173 len 4096 09/30 Read block 507 len 36864 10/30 Write block 193 len 32768 data: 14 14 12 15 18 18 20 19 11/30 Write block 346 len 28672 data: 15 13 11 16 18 18 17 12/30 Write block 347 len 4096 data: 14 13/30 Flush 14/30 Write block 330 len 20480 data: 14 15 16 14 12 15/30 Write block 236 len 24576 data: 18 18 18 16 13 13 16/30 Read block 550 len 36864 17/30 Read block 444 len 28672 18/30 Flush 19/30 Write block 200 len 40960 data: 20 17 20 22 22 20 13 11 10 11 20/30 Write block 183 len 8192 data: 14 13 21/30 Read block 147 len 16384 22/30 Write block 539 len 32768 data: 8 10 10 10 10 10 14 11 23/30 Write block 103 len 24576 data: 14 18 15 17 12 14 24/30 Write block 503 len 4096 data: 15 25/30 Flush 26/30 Write block 436 len 4096 data: 10 27/30 Write block 74 len 16384 data: 14 13 11 11 28/30 Read block 243 len 24576 29/30 Read block 300 len 4096 30/30 Write block 356 len 40960 data: 20 20 20 18 16 18 14 15 12 9 ---------------------------------------------------------------- Crucible gen:164 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Read 9 Done Done Done false 1002 Acked Write 6 Done Done Sent false 1003 Acked Read 10 Done Done Sent false 1004 Acked Read 4 Done Done Sent false 1005 Acked Read 2 Done Done Sent false 1006 Acked Write 8 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Read 9 Done Done Sent false 1009 Acked Write 8 Done Done Sent false 1010 Acked Write 7 Done Done Sent false 1011 Acked Write 1 Done Done Sent false 1012 Acked Flush 0 Done Done Sent false 1013 Acked Write 5 Done Done Sent false 1014 Acked Write 6 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 2 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Write 8 Done Done Sent false 1022 Acked Write 6 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Flush 0 Done Done Sent false 1025 Acked Write 1 Done Done Sent false 1026 Acked Write 4 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 1 Done Done Sent false 1029 Acked Write 10 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 28 30 Done 29 29 2 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) Some(JobId(1024)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:53.749239422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1856} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 159 159 159 313 313 313 T T F <--- 4 080-099 162 162 162 318 318 318 F F F 5 100-119 164 164 161 323 323 316 F F F <--- 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 164 164 162 323 323 318 F F F <--- 10 200-219 164 164 162 323 323 318 F F F <--- 11 220-239 164 164 160 322 322 315 F F F <--- 12 240-259 164 164 163 322 322 320 F F F <--- 13 260-279 162 162 162 318 318 318 F F F 14 280-299 162 162 162 318 318 318 F F F 15 300-319 164 164 158 321 321 311 F F F <--- 16 320-339 164 164 163 322 322 320 F F F <--- 17 340-359 164 164 161 321 321 316 T T F <--- 18 360-379 160 160 160 314 314 314 T T F <--- 19 380-399 162 162 162 319 319 319 F F F 20 400-419 162 162 162 319 319 319 F F F 21 420-439 161 161 161 316 316 316 T T F <--- 22 440-459 154 154 154 304 304 304 F F F 23 460-479 162 162 162 318 318 318 F F F 24 480-499 161 161 161 316 316 316 F F F 25 500-519 164 164 161 323 323 316 F F F <--- 26 520-539 164 164 163 323 323 320 F F F <--- 27 540-559 164 164 163 323 323 320 F F F <--- 28 560-579 163 163 163 320 320 320 F F F 29 580-599 164 164 158 321 321 311 F F T <--- Max gen: 164, Max flush: 323 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 82 Verifying data now No change after verify Loop: 82 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 165 165 165 324 324 324 F F F 4 080-099 162 162 162 318 318 318 F F F 5 100-119 164 164 164 323 323 323 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 164 164 164 323 323 323 F F F 10 200-219 164 164 164 323 323 323 F F F 11 220-239 164 164 164 322 322 322 F F F 12 240-259 164 164 164 322 322 322 F F F 13 260-279 162 162 162 318 318 318 F F F 14 280-299 162 162 162 318 318 318 F F F 15 300-319 164 164 164 321 321 321 F F F 16 320-339 164 164 164 322 322 322 F F F 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 162 162 162 319 319 319 F F F 20 400-419 162 162 162 319 319 319 F F F 21 420-439 165 165 165 324 324 324 F F F 22 440-459 154 154 154 304 304 304 F F F 23 460-479 162 162 162 318 318 318 F F F 24 480-499 161 161 161 316 316 316 F F F 25 500-519 164 164 164 323 323 323 F F F 26 520-539 164 164 164 323 323 323 F F F 27 540-559 164 164 164 323 323 323 F F F 28 560-579 163 163 163 320 320 320 F F F 29 580-599 164 164 164 321 321 321 F F F Max gen: 165, Max flush: 324 Begin loop 83 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.078616282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:58.089332318Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.089392631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.099736209Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100044185Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100084798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100097152Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10012584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} {"msg":"Crucible 530531a3-2592-42d1-a6f9-395edbb3b5f9 has session id: 5bdca49e-8371-42e0-9c20-9db0bda04476","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100478307Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Upstairs opts: Upstairs UUID: 530531a3-2592-42d1-a6f9-395edbb3b5f9, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10051639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Crucible stats registered with UUID: 530531a3-2592-42d1-a6f9-395edbb3b5f9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100533443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:58.100546887Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100728838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100768591Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100784244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100904172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"task":"crutest"} {"msg":"The guest has requested activation with gen:166","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100921235Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} {"msg":"530531a3-2592-42d1-a6f9-395edbb3b5f9 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100937038Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100950063Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"0","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100964126Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"1","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.100993274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"2","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101007048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101021272Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101035545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"ds_connection connected from Ok(127.0.0.1:50945)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101128955Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"2","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"ds_connection connected from Ok(127.0.0.1:39638)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101146917Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"0","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"ds_connection connected from Ok(127.0.0.1:43823)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101257529Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"io task","client":"1","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101845165Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"1","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.101882159Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"0","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10190234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"2","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10227059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102325167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 314, 324, 318, 323, 319, 309, 315, 323, 323, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102366918Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[0]R generation[0..12]: [162, 162, 160, 165, 162, 164, 162, 156, 160, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102384151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102400564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 314, 324, 318, 323, 319, 309, 315, 323, 323, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102416447Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[1]R generation[0..12]: [162, 162, 160, 165, 162, 164, 162, 156, 160, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10243217Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102448463Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 314, 324, 318, 323, 319, 309, 315, 323, 323, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102464046Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[2]R generation[0..12]: [162, 162, 160, 165, 162, 164, 162, 156, 160, 164, 164, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10247971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102496102Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Max found gen is 166","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102509827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Generation requested: 166 >= found:166","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10252386Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Next flush: 325","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102539084Z","hostname"Wait for a query_work_queue command to finish before sending IO :"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102559525Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"":"downstairs","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102574019Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102586823Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"0","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102601417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"1","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102615541Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"client":"2","":"downstairs","session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102629984Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102643159Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"530531a3-2592-42d1-a6f9-395edbb3b5f9 is now active with session: 5bdca49e-8371-42e0-9c20-9db0bda04476","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102661231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102673985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864,"session_id":"5bdca49e-8371-42e0-9c20-9db0bda04476"} {"msg":"The guest has finished waiting for activation with:166","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.10268747Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} {"msg":"Activated sub_volume 530531a3-2592-42d1-a6f9-395edbb3b5f9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:54:58.102698975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 203 len 40960 data: 23 23 21 14 12 11 12 9 9 10 02/30 Read block 430 len 24576 03/30 Write block 429 len 20480 data: 15 13 11 7 6 04/30 Read block 60 len 8192 05/30 Flush 06/30 Flush 07/30 Write block 254 len 32768 data: 13 15 14 17 16 16 15 16 08/30 Read block 138 len 12288 09/30 Write block 567 len 20480 data: 10 12 15 12 9 10/30 Read block 348 len 40960 11/30 Read block 206 len 32768 12/30 Read block 557 len 32768 13/30 Flush 14/30 Flush 15/30 Flush 16/30 Write block 198 len 4096 data: 19 17/30 Write block 442 len 12288 data: 11 12 10 18/30 Read block 80 len 40960 19/30 Write block 508 len 16384 data: 17 16 18 18 20/30 Write block 75 len 36864 data: 14 12 12 11 12 15 13 12 13 21/30 Write block 567 len 40960 data: 11 13 16 13 10 18 18 17 15 12 22/30 Write block 429 len 20480 data: 16 14 12 8 7 23/30 Write block 304 len 8192 data: 9 8 24/30 Read block 267 len 32768 25/30 Read block 537 len 36864 26/30 Write block 279 len 20480 data: 10 8 6 10 9 27/30 Read block 471 len 8192 28/30 Read block 190 len 12288 29/30 Read block 558 len 12288 30/30 Write block 273 len 28672 data: 18 16 13 10 11 10 11 ---------------------------------------------------------------- Crucible gen:166 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 6 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Flush 0 Sent Done Done false 1005 Acked Flush 0 Sent Done Done false 1006 Acked Write 8 Sent Done Done false 1007 Acked Read 3 Sent Done Done false 1008 Acked Write 5 Sent Done Done false 1009 Acked Read 10 Sent Done Done false 1010 Acked Read 8 Sent Done Done false 1011 Acked Read 8 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Flush 0 Sent Done Done false 1015 Acked Write 1 Sent Done Done false 1016 Acked Write 3 Sent Done Done false 1017 Acked Read 10 Sent Done Done false 1018 Acked Write 4 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 5 Sent Done Done false 1022 Acked Write 2 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 9 Sent Done Done false 1025 Acked Write 5 Sent Done Done false 1026 Acked Read 2 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Read 3 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 1 1 28 Done 4 29 29 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1014)) Some(JobId(1014)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:54:58.115254373Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1864} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 165 165 165 324 324 324 F T T <--- 4 080-099 162 162 162 318 318 318 F T T <--- 5 100-119 164 164 164 323 323 323 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 164 164 164 323 323 323 F T T <--- 10 200-219 164 166 166 323 325 325 T F F <--- 11 220-239 164 164 164 322 322 322 F F F 12 240-259 164 166 166 322 327 327 F F F <--- 13 260-279 162 166 166 318 327 327 F T T <--- 14 280-299 162 162 162 318 318 318 F T T <--- 15 300-319 164 164 164 321 321 321 F T T <--- 16 320-339 164 164 164 322 322 322 F F F 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 162 162 162 319 319 319 F F F 20 400-419 162 162 162 319 319 319 F F F 21 420-439 165 166 166 324 325 325 T T T <--- 22 440-459 154 154 154 304 304 304 F T T <--- 23 460-479 162 162 162 318 318 318 F F F 24 480-499 161 161 161 316 316 316 F F F 25 500-519 164 164 164 323 323 323 F T T <--- 26 520-539 164 164 164 323 323 323 F F F 27 540-559 164 164 164 323 323 323 F F F 28 560-579 163 166 166 320 327 327 F T T <--- 29 580-599 164 164 164 321 321 321 F F F Max gen: 166, Max flush: 327 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 83 Verifying data now No change after verify Loop: 83 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 167 167 167 328 328 328 F F F 4 080-099 167 167 167 328 328 328 F F F 5 100-119 164 164 164 323 323 323 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 164 164 164 322 322 322 F F F 12 240-259 166 166 166 327 327 327 F F F 13 260-279 167 167 167 328 328 328 F F F 14 280-299 167 167 167 328 328 328 F F F 15 300-319 167 167 167 328 328 328 F F F 16 320-339 164 164 164 322 322 322 F F F 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 162 162 162 319 319 319 F F F 20 400-419 162 162 162 319 319 319 F F F 21 420-439 167 167 167 328 328 328 F F F 22 440-459 167 167 167 328 328 328 F F F 23 460-479 162 162 162 318 318 318 F F F 24 480-499 161 161 161 316 316 316 F F F 25 500-519 167 167 167 328 328 328 F F F 26 520-539 164 164 164 323 323 323 F F F 27 540-559 164 164 164 323 323 323 F F F 28 560-579 167 167 167 328 328 328 F F F 29 580-599 164 164 164 321 321 321 F F F Max gen: 167, Max flush: 328 Begin loop 84 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.450353994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.462191564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.462513574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.462554687Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.46256934Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.462587362Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} {"msg":"Crucible d8fc40d0-534c-4ece-9976-5614f11b8cb2 has session id: 994451f0-eb34-4a62-a541-fb5425320474","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.462920588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Upstairs opts: Upstairs UUID: d8fc40d0-534c-4ece-9976-5614f11b8cb2, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.462994506Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Crucible stats registered with UUID: d8fc40d0-534c-4ece-9976-5614f11b8cb2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.46303098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:02.463045174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.46321507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463243248Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463258232Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463365755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"task":"crutest"} {"msg":"The guest has requested activation with gen:168","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463383657Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} {"msg":"d8fc40d0-534c-4ece-9976-5614f11b8cb2 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463406447Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463419302Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"0","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463433406Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"1","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.46344725Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"2","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463463603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463478016Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.4634922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"ds_connection connected from Ok(127.0.0.1:63586)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463518819Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"0","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"ds_connection connected from Ok(127.0.0.1:51744)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463533752Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"2","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"ds_connection connected from Ok(127.0.0.1:60662)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.463693293Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"io task","client":"1","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464237587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"2","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464283108Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"1","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464340153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"0","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464637514Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464687152Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 314, 328, 328, 323, 319, 309, 315, 328, 325, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464742668Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[0]R generation[0..12]: [162, 162, 160, 167, 167, 164, 162, 156, 160, 167, 166, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464759711Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464776214Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 314, 328, 328, 323, 319, 309, 315, 328, 325, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464791797Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[1]R generation[0..12]: [162, 162, 160, 167, 167, 164, 162, 156, 160, 167, 166, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464810679Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464827082Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 314, 328, 328, 323, 319, 309, 315, 328, 325, 322]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464842715Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[2]R generation[0..12]: [162, 162, 160, 167, 167, 164, 162, 156, 160, 167, 166, 164]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464858338Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464874481Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Max found gen is 168","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464888405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 168 >= found:168","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464906897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Next flush: 329","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464920501Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464933766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"":"downstairs","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.4649476Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464960264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"0","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464974338Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"client":"1","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.464988572Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 1872,"client":"2","":"downstairs","session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.465013751Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.465029264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"d8fc40d0-534c-4ece-9976-5614f11b8cb2 is now active with session: 994451f0-eb34-4a62-a541-fb5425320474","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.465044158Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.465057172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872,"session_id":"994451f0-eb34-4a62-a541-fb5425320474"} {"msg":"The guest has finished waiting for activation with:168","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.465070686Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} {"msg":"Activated sub_volume d8fc40d0-534c-4ece-9976-5614f11b8cb2","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:02.465081931Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 298 len 16384 data: 15 13 13 14 02/30 Write block 443 len 8192 data: 13 11 03/30 Read block 505 len 16384 04/30 Read block 123 len 24576 05/30 Write block 243 len 16384 data: 13 12 10 10 06/30 Write block 220 len 16384 data: 6 9 11 16 07/30 Write block 517 len 16384 data: 12 13 13 13 08/30 Read block 195 len 28672 09/30 Write block 69 len 8192 data: 7 7 10/30 Read block 415 len 16384 11/30 Read block 69 len 20480 12/30 Write block 422 len 20480 data: 15 15 15 17 14 13/30 Read block 577 len 28672 14/30 Read block 504 len 32768 15/30 Read block 597 len 8192 16/30 Read block 365 len 32768 17/30 Read block 172 len 12288 18/30 Read block 200 len 20480 19/30 Read block 96 len 24576 20/30 Read block 242 len 8192 21/30 Write block 557 len 36864 data: 10 11 12 13 13 13 12 11 12 22/30 Read block 483 len 36864 23/30 Flush 24/30 Write block 442 len 36864 data: 12 14 12 9 9 8 9 8 11 25/30 Write block 86 len 36864 data: 13 12 10 9 10 10 15 16 16 26/30 Write block 584 len 12288 data: 11 12 14 27/30 Write block 388 len 24576 data: 11 10 11 13 17 14 28/30 Read block 124 len 36864 29/30 Write block 588 len 20480 data: 14 16 18 17 14 30/30 Write block 476 len 20480 data: 5 6 11 13 15 ---------------------------------------------------------------- Crucible gen:168 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 4 Done Done Done false 1001 Acked Write 2 Done Done Done false 1002 Acked Read 4 Done Done Done false 1003 Acked Read 6 Done Sent Done false 1004 Acked Write 4 Done Sent Done false 1005 Acked Write 4 Done Sent Done false 1006 Acked Write 4 Done Sent Done false 1007 Acked Read 7 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 4 Done Sent Done false 1010 Acked Read 5 Done Sent Done false 1011 Acked Write 5 Done Sent Done false 1012 Acked Read 7 Done Sent Done false 1013 Acked Read 8 Done Sent Done false 1014 Acked Read 2 Done Sent Done false 1015 Acked Read 8 Done Sent Done false 1016 Acked Read 3 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Read 6 Done Sent Done false 1019 Acked Read 2 Done Sent Done false 1020 Acked Write 9 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Flush 0 Done Sent Done false 1023 Acked Write 9 Done Sent Done false 1024 Acked Write 9 Done Sent Done false 1025 Acked Write 3 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 9 Done Sent Done false 1028 Acked Write 5 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 27 2 31 Done 28 3 28 59 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) None Some(JobId(1022)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:02.47689838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1872} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 168 167 168 329 328 329 F F F <--- 4 080-099 167 167 167 328 328 328 T F T <--- 5 100-119 164 164 164 323 323 323 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 164 168 329 322 329 F F F <--- 12 240-259 168 166 168 329 327 329 F F F <--- 13 260-279 167 167 167 328 328 328 F F F 14 280-299 168 167 168 329 328 329 F T F <--- 15 300-319 168 167 168 329 328 329 F T F <--- 16 320-339 164 164 164 322 322 322 F F F 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 162 162 162 319 319 319 T F T <--- 20 400-419 162 162 162 319 319 319 F F F 21 420-439 168 167 168 329 328 329 F F F <--- 22 440-459 168 167 168 329 328 329 T T T <--- 23 460-479 162 162 162 318 318 318 T F T <--- 24 480-499 161 161 161 316 316 316 T F T <--- 25 500-519 168 167 168 329 328 329 F F F <--- 26 520-539 168 164 168 329 323 329 F F F <--- 27 540-559 168 164 168 329 323 329 F F F <--- 28 560-579 168 167 168 329 328 329 F F F <--- 29 580-599 164 164 164 321 321 321 T F T <--- Max gen: 168, Max flush: 329 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 84 Verifying data now No change after verify Loop: 84 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F F F 3 060-079 168 168 168 329 329 329 F F F 4 080-099 169 169 169 330 330 330 F F F 5 100-119 164 164 164 323 323 323 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 156 156 156 309 309 309 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 167 167 167 328 328 328 F F F 14 280-299 168 168 168 329 329 329 F F F 15 300-319 168 168 168 329 329 329 F F F 16 320-339 164 164 164 322 322 322 F F F 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 169 169 169 330 330 330 F F F 20 400-419 162 162 162 319 319 319 F F F 21 420-439 168 168 168 329 329 329 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 169 169 169 330 330 330 F F F 24 480-499 169 169 169 330 330 330 F F F 25 500-519 168 168 168 329 329 329 F F F 26 520-539 168 168 168 329 329 329 F F F 27 540-559 168 168 168 329 329 329 F F F 28 560-579 168 168 168 329 329 329 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 169, Max flush: 330 Begin loop 85 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.808902281Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:06.819651782Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.81970355Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.830245541Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.830543772Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.830582485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.830594969Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.830612972Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} {"msg":"Crucible b8ed6008-7f75-4b5f-bc4a-de6ab81ee783 has session id: de321043-c2c8-4009-ad54-fd472aba36b8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.830979343Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Upstairs opts: Upstairs UUID: b8ed6008-7f75-4b5f-bc4a-de6ab81ee783, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831011519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Crucible stats registered with UUID: b8ed6008-7f75-4b5f-bc4a-de6ab81ee783","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831028981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:06.831042515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831228465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831253644Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831268568Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831414274Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"task":"crutest"} {"msg":"The guest has requested activation with gen:170","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831432157Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} {"msg":"b8ed6008-7f75-4b5f-bc4a-de6ab81ee783 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.83144694Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831460244Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"0","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831474458Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"1","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831488182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"2","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831501647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.83151606Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831536341Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:49645)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.83158619Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"1","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:42501)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831614248Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"0","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"ds_connection connected from Ok(127.0.0.1:47833)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.831641346Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"io task","client":"2","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.83234794Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"1","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832387802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"0","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832404245Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"2","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832711082Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832741729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[0]R flush_numbers[0..12]: [318, 318, 314, 329, 330, 323, 319, 309, 315, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832782981Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[0]R generation[0..12]: [162, 162, 160, 168, 169, 164, 162, 156, 160, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832800903Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832817546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[1]R flush_numbers[0..12]: [318, 318, 314, 329, 330, 323, 319, 309, 315, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832833289Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[1]R generation[0..12]: [162, 162, 160, 168, 169, 164, 162, 156, 160, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832849013Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832865136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[2]R flush_numbers[0..12]: [318, 318, 314, 329, 330, 323, 319, 309, 315, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832880659Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[2]R generation[0..12]: [162, 162, 160, 168, 169, 164, 162, 156, 160, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832896012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832915054Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Max found gen is 170","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832928868Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Generation requested: 170 >= found:170","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:55:06.832942852Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Next flush: 331","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832960884Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832974088Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"":"downstairs","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.832988022Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833000637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"0","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833014411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"1","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833028255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"client":"2","":"downstairs","session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833042139Z"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833058782Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"b8ed6008-7f75-4b5f-bc4a-de6ab81ee783 is now active with session: de321043-c2c8-4009-ad54-fd472aba36b8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833073885Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.8330865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880,"session_id":"de321043-c2c8-4009-ad54-fd472aba36b8"} {"msg":"The guest has finished waiting for activation with:170","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833099844Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} {"msg":"Activated sub_volume b8ed6008-7f75-4b5f-bc4a-de6ab81ee783","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:06.833111109Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 147 len 20480 data: 14 14 14 13 14 02/30 Read block 141 len 20480 03/30 Read block 408 len 32768 04/30 Read block 473 len 24576 05/30 Read block 471 len 36864 06/30 Read block 503 len 20480 07/30 Write block 407 len 4096 data: 11 08/30 Read block 441 len 28672 09/30 Flush 10/30 Write block 72 len 20480 data: 8 8 15 15 13 11/30 Flush 12/30 Write block 47 len 8192 data: 11 10 13/30 Write block 429 len 16384 data: 17 15 13 9 14/30 Read block 310 len 32768 15/30 Read block 185 len 4096 16/30 Read block 185 len 28672 17/30 Read block 336 len 4096 18/30 Write block 331 len 4096 data: 16 19/30 Write block 93 len 4096 data: 17 20/30 Read block 110 len 20480 21/30 Write block 2 len 24576 data: 6 9 12 14 14 13 22/30 Write block 517 len 36864 data: 13 14 14 14 12 12 9 9 14 23/30 Read block 539 len 32768 24/30 Read block 446 len 12288 25/30 Read block 111 len 4096 26/30 Write block 480 len 40960 data: 16 14 16 14 14 14 14 13 16 17 27/30 Write block 117 len 4096 data: 9 28/30 Read block 233 len 32768 29/30 Write block 302 len 28672 data: 10 10 10 9 9 9 10 30/30 Read block 472 len 20480 ---------------------------------------------------------------- Crucible gen:170 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 5 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 2 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Read 8 Sent Done Done false 1014 Acked Read 1 Sent Done Done false 1015 Acked Read 7 Sent Done Done false 1016 Acked Read 1 Sent Done Done false 1017 Acked Write 1 Sent Done Done false 1018 Acked Write 1 Sent Done Done false 1019 Acked Read 5 Sent Done Done false 1020 Acked Write 6 Sent Done Done false 1021 Acked Write 9 Sent Done Done false 1022 Acked Read 8 Sent Done Done false 1023 Acked Read 3 Sent Done Done false 1024 Acked Read 1 Sent Done Done false 1025 Acked Write 10 Sent Done Done false 1026 Acked Write 1 Sent Done Done false 1027 Acked Read 8 Sent Done Done false 1028 Acked Write 7 Sent Done Done false 1029 Acked Read 5 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 21 0 1 22 Done 0 21 20 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1010)) Some(JobId(1010)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:06.845105721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1880} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 162 162 162 318 318 318 F T T <--- 1 020-039 162 162 162 318 318 318 F F F 2 040-059 160 160 160 314 314 314 F T T <--- 3 060-079 168 170 170 329 332 332 F F F <--- 4 080-099 169 169 169 330 330 330 F T T <--- 5 100-119 164 164 164 323 323 323 F T T <--- 6 120-139 162 162 162 319 319 319 F F F 7 140-159 170 170 170 331 331 331 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 167 167 167 328 328 328 F F F 14 280-299 168 168 168 329 329 329 F F F 15 300-319 168 168 168 329 329 329 F T T <--- 16 320-339 164 164 164 322 322 322 F T T <--- 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 169 169 169 330 330 330 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 168 168 168 329 329 329 F T T <--- 22 440-459 169 169 169 330 330 330 F F F 23 460-479 169 169 169 330 330 330 F F F 24 480-499 169 169 169 330 330 330 F T T <--- 25 500-519 168 168 168 329 329 329 F T T <--- 26 520-539 168 168 168 329 329 329 F T T <--- 27 540-559 168 168 168 329 329 329 F F F 28 560-579 168 168 168 329 329 329 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 170, Max flush: 332 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 85 Verifying data now No change after verify Loop: 85 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 162 162 162 318 318 318 F F F 2 040-059 171 171 171 333 333 333 F F F 3 060-079 170 170 170 332 332 332 F F F 4 080-099 171 171 171 333 333 333 F F F 5 100-119 171 171 171 333 333 333 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 170 170 170 331 331 331 F F F 8 160-179 160 160 160 315 315 315 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 167 167 167 328 328 328 F F F 14 280-299 168 168 168 329 329 329 F F F 15 300-319 171 171 171 333 333 333 F F F 16 320-339 171 171 171 333 333 333 F F F 17 340-359 165 165 165 324 324 324 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 169 169 169 330 330 330 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 171 171 171 333 333 333 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 169 169 169 330 330 330 F F F 24 480-499 171 171 171 333 333 333 F F F 25 500-519 171 171 171 333 333 333 F F F 26 520-539 171 171 171 333 333 333 F F F 27 540-559 168 168 168 329 329 329 F F F 28 560-579 168 168 168 329 329 329 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 171, Max flush: 333 Begin loop 86 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.175072966Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:11.185781385Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.185837041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196186596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196486576Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196527078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196539223Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196557255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} {"msg":"Crucible 7a153259-ff43-4b48-b0c1-9f612d5bc29e has session id: fab5f271-be62-4ed7-9780-1515bb09bd32","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196921227Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Upstairs opts: Upstairs UUID: 7a153259-ff43-4b48-b0c1-9f612d5bc29e, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196956662Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Crucible stats registered with UUID: 7a153259-ff43-4b48-b0c1-9f612d5bc29e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.196972225Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:11.196985599Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197179555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197209352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197224686Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.19733035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"task":"crutest"} {"msg":"The guest has requested activation with gen:172","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197347482Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} {"msg":"7a153259-ff43-4b48-b0c1-9f612d5bc29e active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197364465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197379419Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"0","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197393253Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"1","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197407067Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"2","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197421021Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197435374Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197449638Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"ds_connection connected from Ok(127.0.0.1:49043)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197464692Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"2","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"ds_connection connected from Ok(127.0.0.1:60578)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197479755Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"1","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"ds_connection connected from Ok(127.0.0.1:36029)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.197494599Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"io task","client":"0","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198226102Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"2","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198255849Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"1","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198272242Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"0","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198615853Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198664002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[0]R flush_numbers[0..12]: [333, 318, 333, 332, 333, 333, 319, 331, 315, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198723546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[0]R generation[0..12]: [171, 162, 171, 170, 171, 171, 162, 170, 160, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198743288Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.19876048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[1]R flush_numbers[0..12]: [333, 318, 333, 332, 333, 333, 319, 331, 315, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198776873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[1]R generation[0..12]: [171, 162, 171, 170, 171, 171, 162, 170, 160, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198792526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198809299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[2]R flush_numbers[0..12]: [333, 318, 333, 332, 333, 333, 319, 331, 315, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198825172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[2]R generation[0..12]: [171, 162, 171, 170, 171, 171, 162, 170, 160, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198841535Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198858208Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Max found gen is 172","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198871812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Generation requested: 172 >= found:172","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198886405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Next flush: 334","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198905147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198919071Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"":"downstairs","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198933775Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198946519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"0","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198962323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"1","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.198976716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"client":"2","":"downstairs","session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.19899117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Set Downstairs and Upstairs active after reconciliation"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.199007733Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"7a153259-ff43-4b48-b0c1-9f612d5bc29e is now active with session: fab5f271-be62-4ed7-9780-1515bb09bd32","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.199022726Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.199035331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888,"session_id":"fab5f271-be62-4ed7-9780-1515bb09bd32"} {"msg":"The guest has finished waiting for activation with:172","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.199048405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} {"msg":"Activated sub_volume 7a153259-ff43-4b48-b0c1-9f612d5bc29e","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:11.19906018Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 500 len 40960 data: 16 16 17 16 15 16 18 18 18 17 02/30 Read block 18 len 12288 03/30 Write block 273 len 8192 data: 19 17 04/30 Read block 561 len 40960 05/30 Read block 358 len 8192 06/30 Read block 389 len 40960 07/30 Read block 133 len 24576 08/30 Read block 292 len 4096 09/30 Read block 187 len 16384 10/30 Read block 149 len 28672 11/30 Read block 342 len 12288 12/30 Read block 512 len 16384 13/30 Write block 556 len 16384 data: 10 11 12 13 14/30 Write block 576 len 16384 data: 13 9 12 13 15/30 Write block 158 len 28672 data: 14 15 19 18 18 14 11 16/30 Read block 576 len 8192 17/30 Read block 404 len 32768 18/30 Flush 19/30 Write block 462 len 12288 data: 10 13 15 20/30 Read block 31 len 24576 21/30 Read block 156 len 32768 22/30 Read block 199 len 40960 23/30 Read block 337 len 28672 24/30 Read block 297 len 32768 25/30 Read block 396 len 20480 26/30 Read block 374 len 36864 27/30 Write block 32 len 16384 data: 13 14 14 11 28/30 Write block 43 len 4096 data: 13 29/30 Read block 139 len 28672 30/30 Write block 351 len 28672 data: 19 18 16 15 17 21 21 ---------------------------------------------------------------- Crucible gen:172 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 10 Done Done Done false 1001 Acked Read 3 Done Done Done false 1002 Acked Write 2 Done Done Done false 1003 Acked Read 10 Done Done Done false 1004 Acked Read 2 Done Done Done false 1005 Acked Read 10 Done Done Done false 1006 Acked Read 6 Done Done Done false 1007 Acked Read 1 Sent Done Done false 1008 Acked Read 4 Sent Done Done false 1009 Acked Read 7 Sent Done Done false 1010 Acked Read 3 Sent Done Done false 1011 Acked Read 4 Sent Done Done false 1012 Acked Write 4 Sent Done Done false 1013 Acked Write 4 Sent Done Done false 1014 Acked Write 7 Sent Done Done false 1015 Acked Read 2 Sent Done Done false 1016 Acked Read 8 Sent Done Done false 1017 Acked Flush 0 Sent Done Done false 1018 Acked Write 3 Sent Done Done false 1019 Acked Read 6 Sent Done Done false 1020 Acked Read 8 Sent Done Done false 1021 Acked Read 10 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Read 8 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Read 9 Sent Done Done false 1026 Acked Write 4 Sent Done Done false 1027 Acked Write 1 Sent Done Done false 1028 Acked Read 7 Sent Done Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 23 1 1 25 Done 7 29 29 65 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1017)) Some(JobId(1017)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:11.211276136Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1888} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 162 162 162 318 318 318 F T T <--- 2 040-059 171 171 171 333 333 333 F T T <--- 3 060-079 170 170 170 332 332 332 F F F 4 080-099 171 171 171 333 333 333 F F F 5 100-119 171 171 171 333 333 333 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 170 172 172 331 334 334 F F F <--- 8 160-179 160 172 172 315 334 334 F F F <--- 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 167 172 172 328 334 334 T F F <--- 14 280-299 168 168 168 329 329 329 F F F 15 300-319 171 171 171 333 333 333 F F F 16 320-339 171 171 171 333 333 333 F F F 17 340-359 165 165 165 324 324 324 F T T <--- 18 360-379 165 165 165 324 324 324 F F F 19 380-399 169 169 169 330 330 330 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 171 171 171 333 333 333 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 169 169 169 330 330 330 F T T <--- 24 480-499 171 171 171 333 333 333 F F F 25 500-519 171 172 172 333 334 334 T F F <--- 26 520-539 171 171 171 333 333 333 F F F 27 540-559 168 172 172 329 334 334 F F F <--- 28 560-579 168 172 172 329 334 334 F F F <--- 29 580-599 169 169 169 330 330 330 F F F Max gen: 172, Max flush: 334 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 86 Verifying data now No change after verify Loop: 86 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 173 173 173 335 335 335 F F F 2 040-059 173 173 173 335 335 335 F F F 3 060-079 170 170 170 332 332 332 F F F 4 080-099 171 171 171 333 333 333 F F F 5 100-119 171 171 171 333 333 333 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 172 172 172 334 334 334 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 172 172 172 334 334 334 F F F 14 280-299 168 168 168 329 329 329 F F F 15 300-319 171 171 171 333 333 333 F F F 16 320-339 171 171 171 333 333 333 F F F 17 340-359 173 173 173 335 335 335 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 169 169 169 330 330 330 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 171 171 171 333 333 333 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 173 173 173 335 335 335 F F F 24 480-499 171 171 171 333 333 333 F F F 25 500-519 172 172 172 334 334 334 F F F 26 520-539 171 171 171 333 333 333 F F F 27 540-559 172 172 172 334 334 334 F F F 28 560-579 172 172 172 334 334 334 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 173, Max flush: 335 Begin loop 87 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.537033106Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:15.547715976Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.547770182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558083642Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558372507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.55841157Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558424095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558441987Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} {"msg":"Crucible fe390bf8-b9e2-4096-bb29-8f3e8741a959 has session id: 16c5a87c-4d55-4a63-a9fd-330b5e884db0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558760139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Upstairs opts: Upstairs UUID: fe390bf8-b9e2-4096-bb29-8f3e8741a959, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558790636Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Crucible stats registered with UUID: fe390bf8-b9e2-4096-bb29-8f3e8741a959","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.55880545Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:15.558819134Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.558992968Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559018257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559033621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559160256Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"task":"crutest"} {"msg":"The guest has requested activation with gen:174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559177868Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} {"msg":"fe390bf8-b9e2-4096-bb29-8f3e8741a959 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559199319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559211913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"0","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559226027Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"1","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559239741Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"2","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559253425Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559267599Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559281763Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49504)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.55933537Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"0","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"ds_connection connected from Ok(127.0.0.1:49919)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.559364417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"2","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"ds_connection connected from Ok(127.0.0.1:39979)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.55938086Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"io task","client":"1","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560029379Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"1","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560178105Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"2","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560222725Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"0","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560408135Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560484102Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[0]R flush_numbers[0..12]: [333, 335, 335, 332, 333, 333, 319, 334, 334, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560583499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[0]R generation[0..12]: [171, 173, 173, 170, 171, 171, 162, 172, 172, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560600892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560617504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[1]R flush_numbers[0..12]: [333, 335, 335, 332, 333, 333, 319, 334, 334, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560633297Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[1]R generation[0..12]: [171, 173, 173, 170, 171, 171, 162, 172, 172, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560649221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560665324Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[2]R flush_numbers[0..12]: [333, 335, 335, 332, 333, 333, 319, 334, 334, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560683626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[2]R generation[0..12]: [171, 173, 173, 170, 171, 171, 162, 172, 172, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560699509Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560715682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Wait for a query_work_queue command to finish before sending IO Max found gen is 174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560734004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Generation requested: 174 >= found:174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560748208Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Next flush: 336","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560761432Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560774856Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"":"downstairs","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.56078904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560801954Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"0","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:55:15.560815948Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"1","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560832141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"client":"2","":"downstairs","session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560846285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560859499Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"fe390bf8-b9e2-4096-bb29-8f3e8741a959 is now active with session: 16c5a87c-4d55-4a63-a9fd-330b5e884db0","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560874173Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560886718Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896,"session_id":"16c5a87c-4d55-4a63-a9fd-330b5e884db0"} {"msg":"The guest has finished waiting for activation with:174","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560902051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} {"msg":"Activated sub_volume fe390bf8-b9e2-4096-bb29-8f3e8741a959","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:15.560913596Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 45 len 8192 data: 14 12 02/30 Read block 427 len 4096 03/30 Write block 78 len 28672 data: 12 13 16 14 13 14 14 04/30 Read block 430 len 8192 05/30 Read block 478 len 8192 06/30 Write block 329 len 4096 data: 14 07/30 Read block 571 len 4096 08/30 Read block 261 len 24576 09/30 Flush 10/30 Write block 25 len 20480 data: 13 13 11 14 18 11/30 Read block 149 len 28672 12/30 Write block 306 len 32768 data: 10 10 11 12 10 12 12 8 13/30 Flush 14/30 Flush 15/30 Write block 385 len 32768 data: 11 11 10 12 11 12 14 18 16/30 Read block 95 len 36864 17/30 Read block 45 len 16384 18/30 Read block 407 len 4096 19/30 Read block 389 len 32768 20/30 Write block 421 len 12288 data: 16 16 16 21/30 Read block 439 len 36864 22/30 Write block 61 len 12288 data: 8 9 10 23/30 Read block 318 len 28672 24/30 Write block 316 len 4096 data: 9 25/30 Write block 475 len 40960 data: 11 6 7 12 14 17 15 17 15 15 26/30 Flush 27/30 Write block 57 len 24576 data: 8 9 10 10 9 10 28/30 Read block 387 len 4096 29/30 Write block 525 len 16384 data: 15 12 13 13 30/30 Write block 275 len 32768 data: 14 11 12 11 12 9 7 11 ---------------------------------------------------------------- Crucible gen:174 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 7 Done Done Done false 1003 Acked Read 2 Done Done Done false 1004 Acked Read 2 Sent Done Done false 1005 Acked Write 1 Sent Done Done false 1006 Acked Read 1 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 5 Sent Done Done false 1010 Acked Read 7 Sent Done Done false 1011 Acked Write 8 Sent Done Done false 1012 Acked Flush 0 Sent Done Done false 1013 Acked Flush 0 Sent Done Done false 1014 Acked Write 8 Sent Done Done false 1015 Acked Read 9 Sent Done Done false 1016 Acked Read 4 Sent Done Done false 1017 Acked Read 1 Sent Done Done false 1018 Acked Read 8 Sent Done Done false 1019 Acked Write 3 Sent Done Done false 1020 Acked Read 9 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Read 7 Sent Done Done false 1023 Acked Write 1 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 6 Sent Done Done false 1027 Acked Read 1 Sent Done Done false 1028 Acked Write 4 Sent Sent Done false 1029 Acked Write 8 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 26 2 1 29 Done 4 28 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:15.573307634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1896} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 173 174 174 335 337 337 F F F <--- 2 040-059 173 174 174 335 336 336 T T T <--- 3 060-079 170 174 174 332 339 339 T T T <--- 4 080-099 171 174 174 333 336 336 T F F <--- 5 100-119 171 171 171 333 333 333 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 172 172 172 334 334 334 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 172 172 172 334 334 334 F T T <--- 14 280-299 168 168 168 329 329 329 F T T <--- 15 300-319 171 174 174 333 339 339 F F F <--- 16 320-339 171 174 174 333 336 336 F F F <--- 17 340-359 173 173 173 335 335 335 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 169 174 174 330 339 339 F F F <--- 20 400-419 170 170 170 331 331 331 F F F 21 420-439 171 174 174 333 339 339 F F F <--- 22 440-459 169 169 169 330 330 330 F F F 23 460-479 173 174 174 335 339 339 F F F <--- 24 480-499 171 174 174 333 339 339 F F F <--- 25 500-519 172 172 172 334 334 334 F F F 26 520-539 171 171 171 333 333 333 F T T <--- 27 540-559 172 172 172 334 334 334 F F F 28 560-579 172 172 172 334 334 334 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 174, Max flush: 339 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 87 Verifying data now No change after verify Loop: 87 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 174 174 174 337 337 337 F F F 2 040-059 175 175 175 340 340 340 F F F 3 060-079 175 175 175 340 340 340 F F F 4 080-099 174 174 174 336 336 336 F F F 5 100-119 171 171 171 333 333 333 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 172 172 172 334 334 334 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 167 167 167 328 328 328 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 175 175 175 340 340 340 F F F 14 280-299 175 175 175 340 340 340 F F F 15 300-319 174 174 174 339 339 339 F F F 16 320-339 174 174 174 336 336 336 F F F 17 340-359 173 173 173 335 335 335 F F F 18 360-379 165 165 165 324 324 324 F F F 19 380-399 174 174 174 339 339 339 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 174 174 174 339 339 339 F F F 24 480-499 174 174 174 339 339 339 F F F 25 500-519 172 172 172 334 334 334 F F F 26 520-539 175 175 175 340 340 340 F F F 27 540-559 172 172 172 334 334 334 F F F 28 560-579 172 172 172 334 334 334 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 175, Max flush: 340 Begin loop 88 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.905645511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:19.916369343Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.916420721Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.926733332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927018408Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927058571Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927070626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927088478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} {"msg":"Crucible 2a82a850-7ac7-4715-a36d-f083c690ec73 has session id: f8d781c5-1ea0-41f2-8fb4-a5884c9621a8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927427381Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Upstairs opts: Upstairs UUID: 2a82a850-7ac7-4715-a36d-f083c690ec73, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927472271Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Crucible stats registered with UUID: 2a82a850-7ac7-4715-a36d-f083c690ec73","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927488444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:19.927501799Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927672815Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927700633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927727301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927842161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"task":"crutest"} {"msg":"The guest has requested activation with gen:176","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927859024Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} {"msg":"2a82a850-7ac7-4715-a36d-f083c690ec73 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927876766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927889641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"0","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927903944Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"1","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927917678Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"2","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927931243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927945616Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45213)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.927987298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"2","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.928033118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:61370)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.928049431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"1","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"ds_connection connected from Ok(127.0.0.1:45288)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.928161023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"io task","client":"0","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.928709405Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"2","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.92874578Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"1","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.928976739Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"0","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.92911496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929152033Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[0]R flush_numbers[0..12]: [333, 337, 340, 340, 336, 333, 319, 334, 334, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929342531Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[0]R generation[0..12]: [171, 174, 175, 175, 174, 171, 162, 172, 172, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929360083Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929376376Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[1]R flush_numbers[0..12]: [333, 337, 340, 340, 336, 333, 319, 334, 334, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929392199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[1]R generation[0..12]: [171, 174, 175, 175, 174, 171, 162, 172, 172, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929407563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929423796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[2]R flush_numbers[0..12]: [333, 337, 340, 340, 336, 333, 319, 334, 334, 328, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929439679Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[2]R generation[0..12]: [171, 174, 175, 175, 174, 171, 162, 172, 172, 167, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929455352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929471575Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Max found gen is 176","v":0,"name":"Wait for a query_work_queue command to finish before sending IO crucible","level":30,"time":"2026-03-12T22:55:19.929490937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Generation requested: 176 >= found:176","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929507479Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Next flush: 341","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929520894Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929534258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"":"downstairs","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929548222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929560876Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"0","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.92957486Z","Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"1","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929592203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"client":"2","":"downstairs","session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929606606Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929619951Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"2a82a850-7ac7-4715-a36d-f083c690ec73 is now active with session: f8d781c5-1ea0-41f2-8fb4-a5884c9621a8","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929635004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929647529Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904,"session_id":"f8d781c5-1ea0-41f2-8fb4-a5884c9621a8"} {"msg":"The guest has finished waiting for activation with:176","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929660723Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} {"msg":"Activated sub_volume 2a82a850-7ac7-4715-a36d-f083c690ec73","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:19.929672148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 294 len 12288 data: 14 13 15 03/30 Read block 419 len 8192 04/30 Read block 568 len 8192 05/30 Read block 315 len 28672 06/30 Read block 289 len 32768 07/30 Read block 314 len 40960 08/30 Read block 372 len 24576 09/30 Read block 554 len 32768 10/30 Flush 11/30 Flush 12/30 Write block 70 len 28672 data: 8 7 9 9 16 16 14 13/30 Write block 63 len 8192 data: 11 11 14/30 Read block 70 len 20480 15/30 Write block 539 len 4096 data: 9 16/30 Write block 295 len 28672 data: 14 16 17 16 14 14 15 17/30 Write block 186 len 36864 data: 15 13 11 11 12 14 12 15 15 18/30 Read block 95 len 32768 19/30 Read block 363 len 16384 20/30 Read block 273 len 32768 21/30 Write block 94 len 40960 data: 17 18 16 15 15 14 18 17 15 15 22/30 Write block 54 len 12288 data: 10 11 10 23/30 Flush 24/30 Write block 368 len 24576 data: 12 11 12 9 9 9 25/30 Read block 530 len 20480 26/30 Flush 27/30 Write block 188 len 20480 data: 12 12 13 15 13 28/30 Read block 466 len 24576 29/30 Read block 275 len 24576 30/30 Write block 64 len 24576 data: 12 11 11 10 9 8 ---------------------------------------------------------------- Crucible gen:176 GIO:true work queues: Upstairs:0 downstairs:29 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1001 Acked Write 3 Done Done Done false 1002 Acked Read 2 Done Done Done false 1003 Acked Read 2 Sent Done Done false 1004 Acked Read 7 Sent Done Done false 1005 Acked Read 8 Sent Done Done false 1006 Acked Read 10 Sent Done Done false 1007 Acked Read 6 Sent Done Done false 1008 Acked Read 8 Sent Done Done false 1009 Acked Flush 0 Sent Done Done false 1010 Acked Flush 0 Sent Done Done false 1011 Acked Write 7 Sent Done Done false 1012 Acked Write 2 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Write 1 Sent Done Done false 1015 Acked Write 7 Sent Done Done false 1016 Acked Write 9 Sent Done Done false 1017 Acked Read 8 Sent Done Done false 1018 Acked Read 4 Sent Done Done false 1019 Acked Read 8 Sent Done Done false 1020 Acked Write 10 Sent Done Done false 1021 Acked Write 3 Sent Done Done false 1022 Acked Flush 0 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Read 5 Sent Done Done false 1025 Acked Flush 0 Sent Done Done false 1026 Acked Write 5 Sent Done Done false 1027 Acked Read 6 Sent Done Done false 1028 Acked Read 6 Sent Done Done false 1029 Acked Write 6 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 27 1 1 29 Done 2 28 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1000)) Some(JobId(1025)) Some(JobId(1025)) Downstairs last five completed: 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:19.942143443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1904} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 174 174 174 337 337 337 F F F 2 040-059 175 176 176 340 344 344 F F F <--- 3 060-079 175 176 176 340 344 344 F T T <--- 4 080-099 174 176 176 336 344 344 F F F <--- 5 100-119 171 176 176 333 344 344 F F F <--- 6 120-139 162 162 162 319 319 319 F F F 7 140-159 172 172 172 334 334 334 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 167 176 176 328 344 344 F T T <--- 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 175 175 175 340 340 340 F F F 14 280-299 175 176 176 340 344 344 T F F <--- 15 300-319 174 176 176 339 344 344 F F F <--- 16 320-339 174 174 174 336 336 336 F F F 17 340-359 173 173 173 335 335 335 F F F 18 360-379 165 176 176 324 345 345 F F F <--- 19 380-399 174 174 174 339 339 339 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 174 174 174 339 339 339 F F F 24 480-499 174 174 174 339 339 339 F F F 25 500-519 172 172 172 334 334 334 F F F 26 520-539 175 176 176 340 344 344 F F F <--- 27 540-559 172 172 172 334 334 334 F F F 28 560-579 172 172 172 334 334 334 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 176, Max flush: 345 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 88 Verifying data now No change after verify Loop: 88 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 171 171 333 333 333 F F F 1 020-039 174 174 174 337 337 337 F F F 2 040-059 176 176 176 344 344 344 F F F 3 060-079 177 177 177 346 346 346 F F F 4 080-099 176 176 176 344 344 344 F F F 5 100-119 176 176 176 344 344 344 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 172 172 172 334 334 334 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 177 177 177 346 346 346 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 168 168 329 329 329 F F F 13 260-279 175 175 175 340 340 340 F F F 14 280-299 176 176 176 344 344 344 F F F 15 300-319 176 176 176 344 344 344 F F F 16 320-339 174 174 174 336 336 336 F F F 17 340-359 173 173 173 335 335 335 F F F 18 360-379 176 176 176 345 345 345 F F F 19 380-399 174 174 174 339 339 339 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 174 174 174 339 339 339 F F F 24 480-499 174 174 174 339 339 339 F F F 25 500-519 172 172 172 334 334 334 F F F 26 520-539 176 176 176 344 344 344 F F F 27 540-559 172 172 172 334 334 334 F F F 28 560-579 172 172 172 334 334 334 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 177, Max flush: 346 Begin loop 89 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.262955346Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:24.273696811Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.273748748Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284052422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284398802Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284439705Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284451739Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284469412Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} {"msg":"Crucible 9e71fe6b-ac5d-424c-a6e8-f19af73b0123 has session id: 51cdc2e8-ca34-47d8-9b57-36b3a7a91483","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284835873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Upstairs opts: Upstairs UUID: 9e71fe6b-ac5d-424c-a6e8-f19af73b0123, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.284872607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Crucible stats registered with UUID: 9e71fe6b-ac5d-424c-a6e8-f19af73b0123","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.28488836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:24.284901724Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285110624Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285146198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285161592Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285254332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"task":"crutest"} {"msg":"The guest has requested activation with gen:178","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285271334Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} {"msg":"9e71fe6b-ac5d-424c-a6e8-f19af73b0123 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285293515Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285306299Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"0","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285322002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"1","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285336076Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"2","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.28534957Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285363764Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285378468Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"ds_connection connected from Ok(127.0.0.1:37898)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285393961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"0","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"ds_connection connected from Ok(127.0.0.1:52357)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285409254Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"2","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"ds_connection connected from Ok(127.0.0.1:34815)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.285423778Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"io task","client":"1","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286279987Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"2","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286316031Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"0","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286332714Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"1","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.28668803Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286730332Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[0]R flush_numbers[0..12]: [333, 337, 344, 346, 344, 344, 319, 334, 334, 346, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286792735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[0]R generation[0..12]: [171, 174, 176, 177, 176, 176, 162, 172, 172, 177, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286809817Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286828969Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[1]R flush_numbers[0..12]: [333, 337, 344, 346, 344, 344, 319, 334, 334, 346, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286845012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[1]R generation[0..12]: [171, 174, 176, 177, 176, 176, 162, 172, 172, 177, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286860485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286876558Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[2]R flush_numbers[0..12]: [333, 337, 344, 346, 344, 344, 319, 334, 334, 346, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286892382Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[2]R generation[0..12]: [171, 174, 176, 177, 176, 176, 162, 172, 172, 177, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286907875Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286924598Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Max found gen is 178","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286938472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"Wait for a query_work_queue command to finish before sending IO } {"msg":"Generation requested: 178 >= found:178","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286958823Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Next flush: 347","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286972257Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286985341Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"":"downstairs","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.286999075Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.28701159Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"0","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287025644Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"1","":"downstairs","session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287041587Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"client":"2","":"downstairs","session_id":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287058829Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287072184Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"9e71fe6b-ac5d-424c-a6e8-f19af73b0123 is now active with session: 51cdc2e8-ca34-47d8-9b57-36b3a7a91483","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287087147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287100002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912,"session_id":"51cdc2e8-ca34-47d8-9b57-36b3a7a91483"} {"msg":"The guest has finished waiting for activation with:178","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287113236Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} {"msg":"Activated sub_volume 9e71fe6b-ac5d-424c-a6e8-f19af73b0123","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:24.287124841Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 394 len 12288 data: 10 7 7 02/30 Write block 516 len 28672 data: 14 14 15 15 15 13 13 03/30 Read block 232 len 20480 04/30 Read block 301 len 32768 05/30 Write block 485 len 32768 data: 15 15 14 17 18 17 16 15 06/30 Read block 452 len 32768 07/30 Read block 100 len 28672 08/30 Read block 30 len 28672 09/30 Write block 242 len 8192 data: 12 14 10/30 Read block 8 len 32768 11/30 Read block 173 len 40960 12/30 Flush 13/30 Write block 19 len 28672 data: 16 17 20 20 19 15 14 14/30 Read block 509 len 20480 15/30 Write block 194 len 8192 data: 16 13 16/30 Read block 129 len 32768 17/30 Read block 321 len 24576 18/30 Write block 148 len 36864 data: 15 15 14 15 11 10 13 15 14 19/30 Flush 20/30 Write block 80 len 36864 data: 17 15 14 15 15 14 14 13 11 21/30 Write block 563 len 36864 data: 13 12 13 11 12 14 17 14 11 22/30 Read block 363 len 28672 23/30 Read block 133 len 40960 24/30 Write block 567 len 24576 data: 13 15 18 15 12 19 25/30 Write block 16 len 40960 data: 17 17 15 17 18 21 21 20 16 15 26/30 Write block 342 len 8192 data: 17 17 27/30 Write block 301 len 28672 data: 16 11 11 11 10 11 11 28/30 Write block 245 len 24576 data: 11 11 8 11 11 13 29/30 Read block 423 len 32768 30/30 Write block 373 len 12288 data: 10 10 12 ---------------------------------------------------------------- Crucible gen:178 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 5 Done Done Done false 1003 Acked Read 8 Done Done Done false 1004 Acked Write 8 Done Done Done false 1005 Acked Read 8 Done Done Done false 1006 Acked Read 7 Done Done Done false 1007 Acked Read 7 Done Done Done false 1008 Acked Write 2 Done Done Done false 1009 Acked Read 8 Done Done Done false 1010 Acked Read 10 Sent Done Done false 1011 Acked Flush 0 Sent Done Done false 1012 Acked Write 7 Sent Done Done false 1013 Acked Read 5 Sent Done Done false 1014 Acked Write 2 Sent Done Done false 1015 Acked Read 8 Sent Done Done false 1016 Acked Read 6 Sent Done Done false 1017 Acked Write 9 Sent Done Done false 1018 Acked Flush 0 Sent Done Done false 1019 Acked Write 9 Sent Done Done false 1020 Acked Write 9 Sent Done Done false 1021 Acked Read 7 Sent Done Done false 1022 Acked Read 10 Sent Done Done false 1023 Acked Write 6 Sent Done Done false 1024 Acked Write 10 Sent Done Done false 1025 Acked Write 2 Sent Done Done false 1026 Acked Write 7 Sent Done Done false 1027 Acked Write 6 Sent Done Done false 1028 Acked Read 8 Sent Done Done false 1029 Acked Write 3 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 20 1 1 22 Done 10 29 29 68 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1018)) Some(JobId(1018)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:24.300229541Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1912} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 171 178 178 333 348 348 F T T <--- 1 020-039 174 178 178 337 348 348 F T T <--- 2 040-059 176 176 176 344 344 344 F F F 3 060-079 177 177 177 346 346 346 F F F 4 080-099 176 176 176 344 344 344 F T T <--- 5 100-119 176 176 176 344 344 344 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 172 178 178 334 348 348 F F F <--- 8 160-179 172 172 172 334 334 334 F F F 9 180-199 177 178 178 346 348 348 F F F <--- 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 168 178 178 329 347 347 T T T <--- 13 260-279 175 175 175 340 340 340 F F F 14 280-299 176 176 176 344 344 344 F F F 15 300-319 176 176 176 344 344 344 F T T <--- 16 320-339 174 174 174 336 336 336 F F F 17 340-359 173 173 173 335 335 335 F T T <--- 18 360-379 176 176 176 345 345 345 F T T <--- 19 380-399 174 178 178 339 347 347 T F F <--- 20 400-419 170 170 170 331 331 331 F F F 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 174 174 174 339 339 339 F F F 24 480-499 174 178 178 339 347 347 T F F <--- 25 500-519 172 178 178 334 347 347 T F F <--- 26 520-539 176 178 178 344 347 347 T F F <--- 27 540-559 172 172 172 334 334 334 F F F 28 560-579 172 172 172 334 334 334 F T T <--- 29 580-599 169 169 169 330 330 330 F F F Max gen: 178, Max flush: 348 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 89 Verifying data now No change after verify Loop: 89 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 349 349 349 F F F 1 020-039 179 179 179 349 349 349 F F F 2 040-059 176 176 176 344 344 344 F F F 3 060-079 177 177 177 346 346 346 F F F 4 080-099 179 179 179 349 349 349 F F F 5 100-119 176 176 176 344 344 344 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 178 178 178 348 348 348 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 179 179 179 349 349 349 F F F 13 260-279 175 175 175 340 340 340 F F F 14 280-299 176 176 176 344 344 344 F F F 15 300-319 179 179 179 349 349 349 F F F 16 320-339 174 174 174 336 336 336 F F F 17 340-359 179 179 179 349 349 349 F F F 18 360-379 179 179 179 349 349 349 F F F 19 380-399 178 178 178 347 347 347 F F F 20 400-419 170 170 170 331 331 331 F F F 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 174 174 174 339 339 339 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 178 178 178 347 347 347 F F F 26 520-539 178 178 178 347 347 347 F F F 27 540-559 172 172 172 334 334 334 F F F 28 560-579 179 179 179 349 349 349 F F F 29 580-599 169 169 169 330 330 330 F F F Max gen: 179, Max flush: 349 Begin loop 90 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.639193328Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.651240017Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.651550783Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.651589516Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.65160252Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.651620612Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} {"msg":"Crucible dd06ef7c-bd7f-4f13-8ee7-d8fd869cf02d has session id: 9bf45894-b2a9-44c5-a457-1d62112b072b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652000967Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Upstairs opts: Upstairs UUID: dd06ef7c-bd7f-4f13-8ee7-d8fd869cf02d, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652035722Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Crucible stats registered with UUID: dd06ef7c-bd7f-4f13-8ee7-d8fd869cf02d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652053735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:28.652066959Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652257116Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652293631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652309074Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652413958Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"task":"crutest"} {"msg":"The guest has requested activation with gen:180","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652431081Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} {"msg":"dd06ef7c-bd7f-4f13-8ee7-d8fd869cf02d active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652448873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652461628Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"0","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652475422Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"1","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652489526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"2","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.65250321Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652517384Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652531627Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"ds_connection connected from Ok(127.0.0.1:62055)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652546301Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"1","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"ds_connection connected from Ok(127.0.0.1:47948)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652643639Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"2","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"ds_connection connected from Ok(127.0.0.1:41102)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.652718626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"io task","client":"0","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653225867Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"1","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653254754Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"0","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.65342554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"2","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653601114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653615938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[0]R flush_numbers[0..12]: [349, 349, 344, 346, 349, 344, 319, 348, 334, 348, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653816351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[0]R generation[0..12]: [179, 179, 176, 177, 179, 176, 162, 178, 172, 178, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653833573Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653849766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[1]R flush_numbers[0..12]: [349, 349, 344, 346, 349, 344, 319, 348, 334, 348, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653865409Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[1]R generation[0..12]: [179, 179, 176, 177, 179, 176, 162, 178, 172, 178, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653880843Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653896736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[2]R flush_numbers[0..12]: [349, 349, 344, 346, 349, 344, 319, 348, 334, 348, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653912329Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[2]R generation[0..12]: [179, 179, 176, 177, 179, 176, 162, 178, 172, 178, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653927892Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653946414Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Max found gen is 180","v":0,"name":"crucible","level":30,"time":"Wait for a query_work_queue command to finish before sending IO 2026-03-12T22:55:28.653959828Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Generation requested: 180 >= found:180","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.6539796Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Next flush: 350","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.653993164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654006238Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"":"downstairs","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654020282Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654032877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"0","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654046441Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"1"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654063554Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"client":"2","":"downstairs","session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654078117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654091641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"dd06ef7c-bd7f-4f13-8ee7-d8fd869cf02d is now active with session: 9bf45894-b2a9-44c5-a457-1d62112b072b","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654106625Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654119059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920,"session_id":"9bf45894-b2a9-44c5-a457-1d62112b072b"} {"msg":"The guest has finished waiting for activation with:180","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654132184Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} {"msg":"Activated sub_volume dd06ef7c-bd7f-4f13-8ee7-d8fd869cf02d","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:28.654143479Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 470 len 12288 data: 15 14 13 02/30 Flush 03/30 Write block 404 len 12288 data: 13 14 13 04/30 Read block 40 len 24576 05/30 Write block 321 len 32768 data: 16 14 13 13 13 14 15 18 06/30 Read block 337 len 8192 07/30 Read block 18 len 8192 08/30 Read block 541 len 28672 09/30 Flush 10/30 Write block 590 len 24576 data: 19 18 15 13 12 11 11/30 Read block 25 len 20480 12/30 Write block 528 len 36864 data: 14 16 14 16 16 13 12 10 10 13/30 Write block 79 len 4096 data: 14 14/30 Read block 53 len 16384 15/30 Read block 408 len 32768 16/30 Read block 533 len 36864 17/30 Read block 300 len 28672 18/30 Flush 19/30 Write block 99 len 40960 data: 15 19 18 16 16 19 16 18 13 15 20/30 Write block 13 len 32768 data: 14 17 15 18 18 16 18 19 21/30 Read block 495 len 16384 22/30 Read block 291 len 4096 23/30 Read block 323 len 8192 24/30 Write block 143 len 32768 data: 11 13 13 12 15 16 16 15 25/30 Write block 398 len 36864 data: 9 8 9 9 9 11 14 15 14 26/30 Read block 62 len 36864 27/30 Read block 176 len 16384 28/30 Read block 210 len 24576 29/30 Read block 429 len 12288 30/30 Write block 559 len 36864 data: 14 14 14 14 14 13 14 12 14 ---------------------------------------------------------------- Crucible gen:180 GIO:true work queues: Upstairs:0 downstairs:21 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1009 Acked Write 6 Done Done Done false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 9 Done Done Sent false 1012 Acked Write 1 Done Done Sent false 1013 Acked Read 4 Done Done Sent false 1014 Acked Read 8 Done Done Sent false 1015 Acked Read 9 Done Done Sent false 1016 Acked Read 7 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 10 Done Done Sent false 1019 Acked Write 8 Done Done Sent false 1020 Acked Read 4 Done Done Sent false 1021 Acked Read 1 Done Done Sent false 1022 Acked Read 2 Done Done Sent false 1023 Acked Write 8 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Read 4 Done Done Sent false 1027 Acked Read 6 Done Done Sent false 1028 Acked Read 3 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 20 22 Done 20 20 1 41 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) Some(JobId(1008)) Downstairs last five completed: 1008 1007 1006 1005 1004 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:28.667149042Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1920} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 179 179 179 349 349 349 T T F <--- 1 020-039 179 179 179 349 349 349 T T F <--- 2 040-059 176 176 176 344 344 344 F F F 3 060-079 180 180 177 352 352 346 F F F <--- 4 080-099 179 179 179 349 349 349 T T F <--- 5 100-119 176 176 176 344 344 344 T T F <--- 6 120-139 162 162 162 319 319 319 F F F 7 140-159 178 178 178 348 348 348 T T F <--- 8 160-179 172 172 172 334 334 334 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 179 179 179 349 349 349 F F F 13 260-279 175 175 175 340 340 340 F F F 14 280-299 176 176 176 344 344 344 F F F 15 300-319 179 179 179 349 349 349 F F F 16 320-339 180 180 180 351 351 351 F F F 17 340-359 179 179 179 349 349 349 F F F 18 360-379 179 179 179 349 349 349 F F F 19 380-399 178 178 178 347 347 347 T T F <--- 20 400-419 180 180 180 351 351 351 T T F <--- 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 178 178 178 347 347 347 F F F 26 520-539 180 180 178 352 352 347 F F F <--- 27 540-559 172 172 172 334 334 334 T T F <--- 28 560-579 179 179 179 349 349 349 T T F <--- 29 580-599 180 180 169 352 352 330 F F T <--- Max gen: 180, Max flush: 352 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 90 Verifying data now No change after verify Loop: 90 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 181 181 181 353 353 353 F F F 2 040-059 176 176 176 344 344 344 F F F 3 060-079 180 180 180 352 352 352 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 181 181 181 353 353 353 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 181 181 181 353 353 353 F F F 8 160-179 172 172 172 334 334 334 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 166 166 166 325 325 325 F F F 11 220-239 168 168 168 329 329 329 F F F 12 240-259 179 179 179 349 349 349 F F F 13 260-279 175 175 175 340 340 340 F F F 14 280-299 176 176 176 344 344 344 F F F 15 300-319 179 179 179 349 349 349 F F F 16 320-339 180 180 180 351 351 351 F F F 17 340-359 179 179 179 349 349 349 F F F 18 360-379 179 179 179 349 349 349 F F F 19 380-399 181 181 181 353 353 353 F F F 20 400-419 181 181 181 353 353 353 F F F 21 420-439 174 174 174 339 339 339 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 178 178 178 347 347 347 F F F 26 520-539 180 180 180 352 352 352 F F F 27 540-559 181 181 181 353 353 353 F F F 28 560-579 181 181 181 353 353 353 F F F 29 580-599 180 180 180 352 352 352 F F F Max gen: 181, Max flush: 353 Begin loop 91 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:32.993494626Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005221174Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005537147Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005579258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005591963Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005610175Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} {"msg":"Crucible bdea3a79-4985-4bc4-b1b4-562d7499d1bf has session id: 61664be5-bd79-41ef-a3f6-9764a5f4d85f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005941611Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Upstairs opts: Upstairs UUID: bdea3a79-4985-4bc4-b1b4-562d7499d1bf, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005983263Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Crucible stats registered with UUID: bdea3a79-4985-4bc4-b1b4-562d7499d1bf","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.005998776Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:33.006012211Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006199719Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006236463Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006252197Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006341258Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"task":"crutest"} {"msg":"The guest has requested activation with gen:182","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006358441Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} {"msg":"bdea3a79-4985-4bc4-b1b4-562d7499d1bf active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006375883Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006388378Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"0","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.00640487Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"1","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006419504Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"2","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006433178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006447562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006461806Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"ds_connection connected from Ok(127.0.0.1:63180)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006476879Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"1","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"ds_connection connected from Ok(127.0.0.1:59187)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006491753Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"2","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"ds_connection connected from Ok(127.0.0.1:58733)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.006575527Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"io task","client":"0","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007217278Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"1","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007255182Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"0","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007342134Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"2","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007580631Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007595145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[0]R flush_numbers[0..12]: [353, 353, 344, 352, 353, 353, 319, 353, 334, 348, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007738922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[0]R generation[0..12]: [181, 181, 176, 180, 181, 181, 162, 181, 172, 178, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007756465Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007775716Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[1]R flush_numbers[0..12]: [353, 353, 344, 352, 353, 353, 319, 353, 334, 348, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.00779151Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[1]R generation[0..12]: [181, 181, 176, 180, 181, 181, 162, 181, 172, 178, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007807393Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007823666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[2]R flush_numbers[0..12]: [353, 353, 344, 352, 353, 353, 319, 353, 334, 348, 325, 329]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007839309Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[2]R generation[0..12]: [181, 181, 176, 180, 181, 181, 162, 181, 172, 178, 166, 168]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007854792Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007871055Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Max found gen is 182","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007884779Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"Wait for a query_work_queue command to finish before sending IO downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Generation requested: 182 >= found:182","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.00790491Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Next flush: 354","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007918485Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007932049Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"":"downstairs","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007946352Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007958917Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"0","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007973041Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"1","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.007988564Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"client":"2","":"downstairs","session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.008006117Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.00802004Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"bdea3a79-4985-4bc4-b1b4-562d7499d1bf is now active with session: 61664be5-bd79-41ef-a3f6-9764a5f4d85f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.008035264Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.008047788Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928,"session_id":"61664be5-bd79-41ef-a3f6-9764a5f4d85f"} {"msg":"The guest has finished waiting for activation with:182","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.008061203Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} {"msg":"Activated sub_volume bdea3a79-4985-4bc4-b1b4-562d7499d1bf","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:33.008072838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 168 len 40960 data: 15 16 18 18 16 11 12 13 12 11 02/30 Flush 03/30 Write block 504 len 4096 data: 16 04/30 Flush 05/30 Write block 154 len 20480 data: 14 16 15 13 15 06/30 Write block 245 len 4096 data: 12 07/30 Read block 117 len 12288 08/30 Read block 491 len 4096 09/30 Flush 10/30 Write block 375 len 40960 data: 13 13 13 14 13 18 17 17 14 12 11/30 Read block 182 len 20480 12/30 Write block 110 len 20480 data: 14 13 11 11 8 13/30 Write block 416 len 32768 data: 9 10 12 14 13 17 17 17 14/30 Read block 231 len 32768 15/30 Read block 14 len 24576 16/30 Read block 218 len 20480 17/30 Read block 261 len 32768 18/30 Read block 130 len 8192 19/30 Read block 24 len 28672 20/30 Read block 390 len 12288 21/30 Write block 330 len 12288 data: 15 17 17 22/30 Read block 191 len 32768 23/30 Write block 53 len 16384 data: 9 11 12 11 24/30 Read block 171 len 16384 25/30 Write block 402 len 36864 data: 10 12 15 16 15 12 11 13 12 26/30 Flush 27/30 Write block 217 len 32768 data: 6 7 9 7 10 12 17 14 28/30 Write block 101 len 4096 data: 19 29/30 Write block 264 len 20480 data: 14 15 18 15 14 30/30 Read block 42 len 28672 ---------------------------------------------------------------- Crucible gen:182 GIO:true work queues: Upstairs:0 downstairs:26 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1004 Acked Write 5 Done Done Sent false 1005 Acked Write 1 Done Done Sent false 1006 Acked Read 3 Done Done Sent false 1007 Acked Read 1 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 10 Done Done Sent false 1010 Acked Read 5 Done Done Sent false 1011 Acked Write 5 Done Done Sent false 1012 Acked Write 8 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Read 6 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Read 2 Done Done Sent false 1018 Acked Read 7 Done Done Sent false 1019 Acked Read 3 Done Done Sent false 1020 Acked Write 3 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 4 Done Done Sent false 1024 Acked Write 9 Done Done Sent false 1025 Acked Flush 0 Done Done Sent false 1026 Acked Write 8 Done Done Sent false 1027 Acked Write 1 Done Done Sent false 1028 Acked Write 5 Done Sent Sent false 1029 Acked Read 7 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 2 26 28 Done 26 24 0 50 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1025)) Some(JobId(1025)) Some(JobId(1003)) Downstairs last five completed: 1003 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:33.02045786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1928} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 181 181 181 353 353 353 F F F 2 040-059 182 182 176 357 357 344 F F F <--- 3 060-079 180 180 180 352 352 352 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 182 182 181 357 357 353 T T F <--- 6 120-139 162 162 162 319 319 319 F F F 7 140-159 182 182 181 356 356 353 F F F <--- 8 160-179 182 182 182 354 354 354 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 166 166 166 325 325 325 T T F <--- 11 220-239 168 168 168 329 329 329 T T F <--- 12 240-259 182 182 179 356 356 349 F F F <--- 13 260-279 175 175 175 340 340 340 T T F <--- 14 280-299 176 176 176 344 344 344 F F F 15 300-319 179 179 179 349 349 349 F F F 16 320-339 182 182 180 357 357 351 F F F <--- 17 340-359 179 179 179 349 349 349 F F F 18 360-379 182 182 179 357 357 349 F F F <--- 19 380-399 182 182 181 357 357 353 F F F <--- 20 400-419 182 182 181 357 357 353 F F F <--- 21 420-439 182 182 174 357 357 339 F F F <--- 22 440-459 169 169 169 330 330 330 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 182 182 182 355 355 355 F F F 26 520-539 180 180 180 352 352 352 F F F 27 540-559 181 181 181 353 353 353 F F F 28 560-579 181 181 181 353 353 353 F F F 29 580-599 180 180 180 352 352 352 F F F Max gen: 182, Max flush: 357 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 91 Verifying data now No change after verify Loop: 91 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 181 181 181 353 353 353 F F F 2 040-059 182 182 182 357 357 357 F F F 3 060-079 180 180 180 352 352 352 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 183 183 183 358 358 358 F F F 6 120-139 162 162 162 319 319 319 F F F 7 140-159 182 182 182 356 356 356 F F F 8 160-179 182 182 182 354 354 354 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 183 183 183 358 358 358 F F F 11 220-239 183 183 183 358 358 358 F F F 12 240-259 182 182 182 356 356 356 F F F 13 260-279 183 183 183 358 358 358 F F F 14 280-299 176 176 176 344 344 344 F F F 15 300-319 179 179 179 349 349 349 F F F 16 320-339 182 182 182 357 357 357 F F F 17 340-359 179 179 179 349 349 349 F F F 18 360-379 182 182 182 357 357 357 F F F 19 380-399 182 182 182 357 357 357 F F F 20 400-419 182 182 182 357 357 357 F F F 21 420-439 182 182 182 357 357 357 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 182 182 182 355 355 355 F F F 26 520-539 180 180 180 352 352 352 F F F 27 540-559 181 181 181 353 353 353 F F F 28 560-579 181 181 181 353 353 353 F F F 29 580-599 180 180 180 352 352 352 F F F Max gen: 183, Max flush: 358 Begin loop 92 Downstairs to restart: 0 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.341960983Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"task":"crutest"} {"msg":"Failed to get info from \"http://127.0.0.1:12810\" Communication Error: error sending request for url (http://127.0.0.1:12810/region-info)","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:37.352826724Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"task":"crutest"} {"msg":"look at: http://127.0.0.1:12820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.352880621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008820, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.363296727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.363592009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.363628793Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.363641457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.363659189Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} {"msg":"Crucible 4b463fb1-c4fe-439b-baf9-23fad00b67e4 has session id: 5f6b08a8-7b4d-4731-8972-387c918784b5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364015525Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Upstairs opts: Upstairs UUID: 4b463fb1-c4fe-439b-baf9-23fad00b67e4, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.36405003Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Crucible stats registered with UUID: 4b463fb1-c4fe-439b-baf9-23fad00b67e4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364067812Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:37.364080847Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364236859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364262218Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364277052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364431095Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"task":"crutest"} {"msg":"The guest has requested activation with gen:184","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364448887Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} {"msg":"4b463fb1-c4fe-439b-baf9-23fad00b67e4 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364463791Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364476255Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"0","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364490319Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"1","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364504433Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"2","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364517987Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364538248Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364552682Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:47114)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364612806Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"2","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:52087)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364643433Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"1","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"ds_connection connected from Ok(127.0.0.1:59759)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.364727766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"io task","client":"0","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365363901Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"2","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365420586Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"1","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365439168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"0","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365719526Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365791455Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[0]R flush_numbers[0..12]: [353, 353, 357, 352, 353, 358, 319, 356, 354, 348, 358, 358]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365855478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[0]R generation[0..12]: [181, 181, 182, 180, 181, 183, 162, 182, 182, 178, 183, 183]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.36587305Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365889583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[1]R flush_numbers[0..12]: [353, 353, 357, 352, 353, 358, 319, 356, 354, 348, 358, 358]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365905496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[1]R generation[0..12]: [181, 181, 182, 180, 181, 183, 162, 182, 182, 178, 183, 183]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365921569Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365937852Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[2]R flush_numbers[0..12]: [353, 353, 357, 352, 353, 358, 319, 356, 354, 348, 358, 358]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365954595Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[2]R generation[0..12]: [181, 181, 182, 180, 181, 183, 162, 182, 182, 178, 183, 183]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365974666Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.365991419Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Max found gen is 184","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366005503Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Wait for a query_work_queue command to finish before sending IO Generation requested: 184 >= found:184","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366025824Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Next flush: 359","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366039518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366053552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"":"downstairs","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366067985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.36608081Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"0","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366095114Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"1","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366108928Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"client":"2","":"downstairs","session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:55:37.366123761Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366138825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"4b463fb1-c4fe-439b-baf9-23fad00b67e4 is now active with session: 5f6b08a8-7b4d-4731-8972-387c918784b5","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366154178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366166813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936,"session_id":"5f6b08a8-7b4d-4731-8972-387c918784b5"} {"msg":"The guest has finished waiting for activation with:184","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366180407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} {"msg":"Activated sub_volume 4b463fb1-c4fe-439b-baf9-23fad00b67e4","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:37.366191842Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 145 len 12288 data: 14 13 16 02/30 Write block 73 len 24576 data: 10 17 17 15 13 13 03/30 Flush 04/30 Flush 05/30 Write block 562 len 8192 data: 15 15 06/30 Read block 455 len 4096 07/30 Write block 373 len 20480 data: 11 11 14 14 14 08/30 Read block 356 len 28672 09/30 Flush 10/30 Write block 168 len 32768 data: 16 17 19 19 17 12 13 14 11/30 Read block 497 len 4096 12/30 Write block 137 len 24576 data: 14 14 14 14 15 15 13/30 Read block 259 len 32768 14/30 Write block 286 len 8192 data: 23 24 15/30 Read block 459 len 8192 16/30 Flush 17/30 Flush 18/30 Write block 38 len 20480 data: 11 11 12 11 14 19/30 Read block 305 len 36864 20/30 Read block 90 len 16384 21/30 Write block 246 len 8192 data: 12 9 22/30 Write block 301 len 16384 data: 17 12 12 12 23/30 Write block 405 len 4096 data: 17 24/30 Write block 366 len 32768 data: 11 11 13 12 13 10 10 12 25/30 Read block 238 len 8192 26/30 Read block 208 len 32768 27/30 Read block 424 len 12288 28/30 Read block 406 len 12288 29/30 Write block 359 len 8192 data: 19 17 30/30 Read block 3 len 24576 ---------------------------------------------------------------- Crucible gen:184 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 3 Sent Done Done false 1001 Acked Write 6 Sent Done Done false 1002 Acked Flush 0 Sent Done Done false 1003 Acked Flush 0 Sent Done Done false 1004 Acked Write 2 Sent Done Done false 1005 Acked Read 1 Sent Done Done false 1006 Acked Write 5 Sent Done Done false 1007 Acked Read 7 Sent Done Done false 1008 Acked Flush 0 Sent Done Done false 1009 Acked Write 8 Sent Done Done false 1010 Acked Read 1 Sent Done Done false 1011 Acked Write 6 Sent Done Done false 1012 Acked Read 8 Sent Done Done false 1013 Acked Write 2 Sent Done Done false 1014 Acked Read 2 Sent Done Done false 1015 Acked Flush 0 Sent Done Done false 1016 Acked Flush 0 Sent Done Done false 1017 Acked Write 5 Sent Done Done false 1018 Acked Read 9 Sent Done Done false 1019 Acked Read 4 Sent Done Done false 1020 Acked Write 2 Sent Done Done false 1021 Acked Write 4 Sent Done Done false 1022 Acked Write 1 Sent Done Done false 1023 Acked Write 8 Sent Done Done false 1024 Acked Read 2 Sent Done Done false 1025 Acked Read 8 Sent Done Done false 1026 Acked Read 3 Sent Done Done false 1027 Acked Read 3 Sent Done Done false 1028 Acked Write 2 Sent Done Sent false 1029 Acked Read 6 Sent Done Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 30 0 2 32 Done 0 30 28 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: None Some(JobId(1016)) Some(JobId(1016)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:37.377576998Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1936} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 181 181 181 353 353 353 F T T <--- 2 040-059 182 182 182 357 357 357 F T T <--- 3 060-079 180 184 184 352 359 359 F F F <--- 4 080-099 181 181 181 353 353 353 F F F 5 100-119 183 183 183 358 358 358 F F F 6 120-139 162 184 184 319 362 362 F F F <--- 7 140-159 182 184 184 356 362 362 F F F <--- 8 160-179 182 184 184 354 362 362 F F F <--- 9 180-199 178 178 178 348 348 348 F F F 10 200-219 183 183 183 358 358 358 F F F 11 220-239 183 183 183 358 358 358 F F F 12 240-259 182 182 182 356 356 356 F T T <--- 13 260-279 183 183 183 358 358 358 F F F 14 280-299 176 184 184 344 362 362 F F F <--- 15 300-319 179 179 179 349 349 349 F T T <--- 16 320-339 182 182 182 357 357 357 F F F 17 340-359 179 179 179 349 349 349 F T T <--- 18 360-379 182 184 184 357 361 361 F T T <--- 19 380-399 182 182 182 357 357 357 F F F 20 400-419 182 182 182 357 357 357 F T T <--- 21 420-439 182 182 182 357 357 357 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 182 182 182 355 355 355 F F F 26 520-539 180 180 180 352 352 352 F F F 27 540-559 181 181 181 353 353 353 F F F 28 560-579 181 184 184 353 361 361 F F F <--- 29 580-599 180 180 180 352 352 352 F F F Max gen: 184, Max flush: 362 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 92 Verifying data now No change after verify Loop: 92 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 185 185 185 363 363 363 F F F 2 040-059 185 185 185 363 363 363 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 183 183 183 358 358 358 F F F 6 120-139 184 184 184 362 362 362 F F F 7 140-159 184 184 184 362 362 362 F F F 8 160-179 184 184 184 362 362 362 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 183 183 183 358 358 358 F F F 11 220-239 183 183 183 358 358 358 F F F 12 240-259 185 185 185 363 363 363 F F F 13 260-279 183 183 183 358 358 358 F F F 14 280-299 184 184 184 362 362 362 F F F 15 300-319 185 185 185 363 363 363 F F F 16 320-339 182 182 182 357 357 357 F F F 17 340-359 185 185 185 363 363 363 F F F 18 360-379 185 185 185 363 363 363 F F F 19 380-399 182 182 182 357 357 357 F F F 20 400-419 185 185 185 363 363 363 F F F 21 420-439 182 182 182 357 357 357 F F F 22 440-459 169 169 169 330 330 330 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 182 182 182 355 355 355 F F F 26 520-539 180 180 180 352 352 352 F F F 27 540-559 181 181 181 353 353 353 F F F 28 560-579 184 184 184 361 361 361 F F F 29 580-599 180 180 180 352 352 352 F F F Max gen: 185, Max flush: 363 Begin loop 93 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.712864387Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.724808421Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725162347Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725203709Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725216764Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725234996Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} {"msg":"Crucible 5e4723df-31a9-4df9-b92a-7f1a8fd0821a has session id: 509cdc9d-e144-4f7d-94e9-5ba2fdea8cab","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.72557112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Upstairs opts: Upstairs UUID: 5e4723df-31a9-4df9-b92a-7f1a8fd0821a, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725605315Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Crucible stats registered with UUID: 5e4723df-31a9-4df9-b92a-7f1a8fd0821a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725621628Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:41.725635162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725821112Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725846241Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.725860685Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726008661Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"task":"crutest"} {"msg":"The guest has requested activation with gen:186","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726022555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} {"msg":"5e4723df-31a9-4df9-b92a-7f1a8fd0821a active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726036758Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726049813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"0","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726063707Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"1","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726077611Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"2","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726091555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726105848Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726127949Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"ds_connection connected from Ok(127.0.0.1:43186)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726144452Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"2","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"ds_connection connected from Ok(127.0.0.1:40913)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726159735Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"1","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"ds_connection connected from Ok(127.0.0.1:62116)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726270677Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"io task","client":"0","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726847427Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"1","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726883411Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"0","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.726967985Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"2","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727281889Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727307298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[0]R flush_numbers[0..12]: [353, 363, 363, 359, 353, 358, 362, 362, 362, 348, 358, 358]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727382585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[0]R generation[0..12]: [181, 185, 185, 184, 181, 183, 184, 184, 184, 178, 183, 183]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727405395Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727422228Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[1]R flush_numbers[0..12]: [353, 363, 363, 359, 353, 358, 362, 362, 362, 348, 358, 358]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727438111Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[1]R generation[0..12]: [181, 185, 185, 184, 181, 183, 184, 184, 184, 178, 183, 183]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727453614Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727470877Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[2]R flush_numbers[0..12]: [353, 363, 363, 359, 353, 358, 362, 362, 362, 348, 358, 358]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727487Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[2]R generation[0..12]: [181, 185, 185, 184, 181, 183, 184, 184, 184, 178, 183, 183]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727502603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727519026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Max found gen is 186","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.72753266Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Generation requested: 186 >= found:186Wait for a query_work_queue command to finish before sending IO ","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727553321Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Next flush: 364","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727567105Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727580249Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"":"downstairs","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727594563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727607298Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"0","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727623231Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"1","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727637145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"client":"2","":"downstairs","session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727654367Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727667921Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"5e4723df-31a9-4df9-b92a-7f1a8fd0821a is now active with session: 509cdc9d-e144-4f7d-94e9-5ba2fdea8cab","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727683285Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727695749Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944,"session_id":"509cdc9d-e144-4f7d-94e9-5ba2fdea8cab"} {"msg":"The guest has finished waiting for activation with:186","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727709104Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} {"msg":"Activated sub_volume 5e4723df-31a9-4df9-b92a-7f1a8fd0821a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:41.727720439Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 552 len 4096 data: 7 02/30 Write block 215 len 28672 data: 9 7 7 8 10 8 11 03/30 Read block 282 len 36864 04/30 Write block 569 len 12288 data: 19 16 13 05/30 Read block 315 len 32768 06/30 Read block 267 len 24576 07/30 Read block 369 len 20480 08/30 Write block 125 len 32768 data: 17 15 14 12 12 10 9 13 09/30 Write block 535 len 40960 data: 11 11 8 7 10 11 11 11 11 11 10/30 Write block 286 len 12288 data: 24 25 21 11/30 Read block 590 len 28672 12/30 Read block 59 len 16384 13/30 Read block 547 len 8192 14/30 Read block 353 len 32768 15/30 Write block 314 len 16384 data: 7 8 10 11 16/30 Read block 449 len 20480 17/30 Write block 547 len 28672 data: 14 16 14 15 11 8 7 18/30 Write block 244 len 24576 data: 13 13 13 10 12 12 19/30 Read block 69 len 16384 20/30 Flush 21/30 Write block 119 len 32768 data: 9 10 12 15 15 16 18 16 22/30 Write block 439 len 28672 data: 15 14 13 13 15 13 10 23/30 Flush 24/30 Write block 290 len 4096 data: 20 25/30 Write block 591 len 8192 data: 19 16 26/30 Read block 278 len 20480 27/30 Write block 381 len 24576 data: 18 18 15 13 12 12 28/30 Write block 55 len 12288 data: 13 12 9 29/30 Write block 121 len 28672 data: 13 16 16 17 19 17 15 30/30 Read block 57 len 16384 ---------------------------------------------------------------- Crucible gen:186 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 1 Done Done Done false 1001 Acked Write 7 Done Done Done false 1002 Acked Read 9 Done Done Done false 1003 Acked Write 3 Done Done Sent false 1004 Acked Read 8 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Read 5 Done Done Sent false 1007 Acked Write 8 Done Done Sent false 1008 Acked Write 10 Done Done Sent false 1009 Acked Write 3 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Read 4 Done Done Sent false 1012 Acked Read 2 Done Done Sent false 1013 Acked Read 8 Done Done Sent false 1014 Acked Write 4 Done Done Sent false 1015 Acked Read 5 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 6 Done Done Sent false 1018 Acked Read 4 Done Done Sent false 1019 Acked Flush 0 Done Done Sent false 1020 Acked Write 8 Done Done Sent false 1021 Acked Write 7 Done Done Sent false 1022 Acked Flush 0 Done Done Sent false 1023 Acked Write 1 Done Done Sent false 1024 Acked Write 2 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Write 3 Done Done Sent false 1028 Acked Write 7 Done Done Sent false 1029 Acked Read 4 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 27 28 Done 30 29 3 62 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1022)) Some(JobId(1022)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:41.739519825Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1944} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 185 185 185 363 363 363 F F F 2 040-059 185 185 185 363 363 363 T T F <--- 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 186 186 183 365 365 358 F F F <--- 6 120-139 186 186 184 365 365 362 T T F <--- 7 140-159 184 184 184 362 362 362 F F F 8 160-179 184 184 184 362 362 362 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 186 186 183 364 364 358 F F T <--- 11 220-239 186 186 183 364 364 358 F F T <--- 12 240-259 186 186 185 364 364 363 F F F <--- 13 260-279 183 183 183 358 358 358 F F F 14 280-299 186 186 184 364 364 362 T T F <--- 15 300-319 186 186 185 364 364 363 F F F <--- 16 320-339 182 182 182 357 357 357 F F F 17 340-359 185 185 185 363 363 363 F F F 18 360-379 185 185 185 363 363 363 F F F 19 380-399 182 182 182 357 357 357 T T F <--- 20 400-419 185 185 185 363 363 363 F F F 21 420-439 186 186 182 365 365 357 F F F <--- 22 440-459 186 186 169 365 365 330 F F F <--- 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 182 182 182 355 355 355 F F F 26 520-539 186 186 180 364 364 352 F F F <--- 27 540-559 186 186 181 364 364 353 F F T <--- 28 560-579 186 186 184 364 364 361 F F F <--- 29 580-599 180 180 180 352 352 352 T T F <--- Max gen: 186, Max flush: 365 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 93 Verifying data now No change after verify Loop: 93 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 185 185 185 363 363 363 F F F 2 040-059 187 187 187 366 366 366 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 186 186 186 365 365 365 F F F 6 120-139 187 187 187 366 366 366 F F F 7 140-159 184 184 184 362 362 362 F F F 8 160-179 184 184 184 362 362 362 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 186 186 186 364 364 364 F F F 11 220-239 186 186 186 364 364 364 F F F 12 240-259 186 186 186 364 364 364 F F F 13 260-279 183 183 183 358 358 358 F F F 14 280-299 187 187 187 366 366 366 F F F 15 300-319 186 186 186 364 364 364 F F F 16 320-339 182 182 182 357 357 357 F F F 17 340-359 185 185 185 363 363 363 F F F 18 360-379 185 185 185 363 363 363 F F F 19 380-399 187 187 187 366 366 366 F F F 20 400-419 185 185 185 363 363 363 F F F 21 420-439 186 186 186 365 365 365 F F F 22 440-459 186 186 186 365 365 365 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 178 178 178 347 347 347 F F F 25 500-519 182 182 182 355 355 355 F F F 26 520-539 186 186 186 364 364 364 F F F 27 540-559 186 186 186 364 364 364 F F F 28 560-579 186 186 186 364 364 364 F F F 29 580-599 187 187 187 366 366 366 F F F Max gen: 187, Max flush: 366 Begin loop 94 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.069489588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.081435731Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.08175971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.081797294Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.081809888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.081827801Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} {"msg":"Crucible 64260433-94b7-4bc4-a9ed-f41be50a15ad has session id: ef318141-147f-4038-b285-f5202591f8a9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082150351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Upstairs opts: Upstairs UUID: 64260433-94b7-4bc4-a9ed-f41be50a15ad, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082186865Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Crucible stats registered with UUID: 64260433-94b7-4bc4-a9ed-f41be50a15ad","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082203198Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:46.082216922Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082427501Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082462655Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082478149Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082623066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"task":"crutest"} {"msg":"The guest has requested activation with gen:188","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082640448Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} {"msg":"64260433-94b7-4bc4-a9ed-f41be50a15ad active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082656032Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082669006Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"0","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.08268306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"1","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082697164Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"2","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082711118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082725192Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082739496Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:37463)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082766454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"0","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:33585)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082781827Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"2","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"ds_connection connected from Ok(127.0.0.1:58846)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.082796531Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"io task","client":"1","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.083512111Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"1","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.083591776Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"0","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.083609608Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"2","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.0839295Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.083978359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[0]R flush_numbers[0..12]: [353, 363, 366, 359, 353, 365, 366, 362, 362, 348, 364, 364]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084048048Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[0]R generation[0..12]: [181, 185, 187, 184, 181, 186, 187, 184, 184, 178, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084065551Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084082254Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[1]R flush_numbers[0..12]: [353, 363, 366, 359, 353, 365, 366, 362, 362, 348, 364, 364]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084098427Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[1]R generation[0..12]: [181, 185, 187, 184, 181, 186, 187, 184, 184, 178, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.0841141Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084130473Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[2]R flush_numbers[0..12]: [353, 363, 366, 359, 353, 365, 366, 362, 362, 348, 364, 364]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084148785Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[2]R generation[0..12]: [181, 185, 187, 184, 181, 186, 187, 184, 184, 178, 186, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084164518Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084180821Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Max found gen is 188","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:55:46.084203331Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Generation requested: 188 >= found:188","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084221813Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Next flush: 367","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084235337Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084248961Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"":"downstairs","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084262845Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.08427562Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"0","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084289904Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"1","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084305667Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"client":"2","":"downstairs","session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.08432037Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084334014Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"64260433-94b7-4bc4-a9ed-f41be50a15ad is now active with session: ef318141-147f-4038-b285-f5202591f8a9","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084349278Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084362012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952,"session_id":"ef318141-147f-4038-b285-f5202591f8a9"} {"msg":"The guest has finished waiting for activation with:188","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084377016Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} {"msg":"Activated sub_volume 64260433-94b7-4bc4-a9ed-f41be50a15ad","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:46.084388501Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 26 len 28672 data: 14 12 15 19 16 15 14 02/30 Flush 03/30 Write block 498 len 24576 data: 18 21 17 17 18 17 04/30 Write block 411 len 32768 data: 14 11 12 11 10 10 11 13 05/30 Write block 404 len 28672 data: 16 18 16 13 12 14 13 06/30 Read block 472 len 36864 07/30 Write block 357 len 4096 data: 22 08/30 Read block 388 len 40960 09/30 Read block 185 len 32768 10/30 Read block 448 len 32768 11/30 Read block 424 len 28672 12/30 Write block 122 len 24576 data: 17 17 18 20 18 16 13/30 Read block 366 len 16384 14/30 Write block 47 len 32768 data: 12 11 13 13 13 10 10 12 15/30 Read block 287 len 8192 16/30 Write block 246 len 8192 data: 14 11 17/30 Write block 451 len 28672 data: 13 15 16 15 13 15 13 18/30 Write block 298 len 28672 data: 17 15 15 18 13 13 13 19/30 Read block 520 len 40960 20/30 Write block 207 len 24576 data: 13 12 13 10 10 11 21/30 Read block 44 len 36864 22/30 Read block 171 len 40960 23/30 Write block 344 len 12288 data: 17 16 16 24/30 Flush 25/30 Write block 568 len 24576 data: 16 20 17 14 20 19 26/30 Read block 481 len 20480 27/30 Read block 451 len 24576 28/30 Write block 247 len 32768 data: 12 13 13 14 13 12 14 14 29/30 Write block 419 len 36864 data: 15 14 18 18 18 16 18 15 11 30/30 Read block 325 len 12288 ---------------------------------------------------------------- Crucible gen:188 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 7 Done Done Done false 1001 Acked Flush 0 Done Done Sent false 1002 Acked Write 6 Done Done Sent false 1003 Acked Write 8 Done Done Sent false 1004 Acked Write 7 Done Done Sent false 1005 Acked Read 9 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Read 10 Done Done Sent false 1008 Acked Read 8 Done Done Sent false 1009 Acked Read 8 Done Done Sent false 1010 Acked Read 7 Done Done Sent false 1011 Acked Write 6 Done Done Sent false 1012 Acked Read 4 Done Done Sent false 1013 Acked Write 8 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 2 Done Done Sent false 1016 Acked Write 7 Done Done Sent false 1017 Acked Write 7 Done Done Sent false 1018 Acked Read 10 Done Done Sent false 1019 Acked Write 6 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Read 10 Done Done Sent false 1022 Acked Write 3 Done Done Sent false 1023 Acked Flush 0 Done Done Sent false 1024 Acked Write 6 Done Done Sent false 1025 Acked Read 5 Done Done Sent false 1026 Acked Read 6 Done Done Sent false 1027 Acked Write 8 Done Done Sent false 1028 Acked Write 9 Done Done Sent false 1029 Acked Read 3 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 29 30 Done 30 29 1 60 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1023)) Some(JobId(1023)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:46.097595957Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1952} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 188 188 185 367 367 363 F F T <--- 2 040-059 188 188 187 368 368 366 F F F <--- 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 186 186 186 365 365 365 F F F 6 120-139 188 188 187 368 368 366 F F F <--- 7 140-159 184 184 184 362 362 362 F F F 8 160-179 184 184 184 362 362 362 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 188 188 186 368 368 364 F F F <--- 11 220-239 186 186 186 364 364 364 F F F 12 240-259 188 188 186 368 368 364 T T F <--- 13 260-279 183 183 183 358 358 358 F F F 14 280-299 188 188 187 368 368 366 F F F <--- 15 300-319 188 188 186 368 368 364 F F F <--- 16 320-339 182 182 182 357 357 357 F F F 17 340-359 188 188 185 368 368 363 F F F <--- 18 360-379 185 185 185 363 363 363 F F F 19 380-399 187 187 187 366 366 366 F F F 20 400-419 188 188 185 368 368 363 T T F <--- 21 420-439 186 186 186 365 365 365 T T F <--- 22 440-459 188 188 186 368 368 365 F F F <--- 23 460-479 180 180 180 350 350 350 F F F 24 480-499 188 188 178 368 368 347 F F F <--- 25 500-519 188 188 182 368 368 355 F F F <--- 26 520-539 186 186 186 364 364 364 F F F 27 540-559 186 186 186 364 364 364 F F F 28 560-579 186 186 186 364 364 364 T T F <--- 29 580-599 187 187 187 366 366 366 F F F Max gen: 188, Max flush: 368 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 94 Verifying data now No change after verify Loop: 94 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 188 188 188 367 367 367 F F F 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 186 186 186 365 365 365 F F F 6 120-139 188 188 188 368 368 368 F F F 7 140-159 184 184 184 362 362 362 F F F 8 160-179 184 184 184 362 362 362 F F F 9 180-199 178 178 178 348 348 348 F F F 10 200-219 188 188 188 368 368 368 F F F 11 220-239 186 186 186 364 364 364 F F F 12 240-259 189 189 189 369 369 369 F F F 13 260-279 183 183 183 358 358 358 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 182 182 182 357 357 357 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 185 185 185 363 363 363 F F F 19 380-399 187 187 187 366 366 366 F F F 20 400-419 189 189 189 369 369 369 F F F 21 420-439 189 189 189 369 369 369 F F F 22 440-459 188 188 188 368 368 368 F F F 23 460-479 180 180 180 350 350 350 F F F 24 480-499 188 188 188 368 368 368 F F F 25 500-519 188 188 188 368 368 368 F F F 26 520-539 186 186 186 364 364 364 F F F 27 540-559 186 186 186 364 364 364 F F F 28 560-579 189 189 189 369 369 369 F F F 29 580-599 187 187 187 366 366 366 F F F Max gen: 189, Max flush: 369 Begin loop 95 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.424124012Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.435939092Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436284052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436325634Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436338009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436355651Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} {"msg":"Crucible f1234a74-5966-48e2-903a-d50382b52b2f has session id: 92e54e99-5549-4a0a-a573-9e571f167a1a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436746872Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Upstairs opts: Upstairs UUID: f1234a74-5966-48e2-903a-d50382b52b2f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436782546Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Crucible stats registered with UUID: f1234a74-5966-48e2-903a-d50382b52b2f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436798469Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:50.436811833Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.436982939Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437020873Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437036146Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437164161Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"task":"crutest"} {"msg":"The guest has requested activation with gen:190","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437181443Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} {"msg":"f1234a74-5966-48e2-903a-d50382b52b2f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437199315Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.43721213Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"0","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437226284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"1","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437240388Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"2","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437254572Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437269085Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437283349Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:44592)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.43732669Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"0","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52318)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437354418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"2","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"ds_connection connected from Ok(127.0.0.1:52576)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.437370351Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"io task","client":"1","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438083552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"1","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.43811275Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"0","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438130002Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"2","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438493714Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438548621Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[0]R flush_numbers[0..12]: [353, 367, 368, 359, 353, 365, 368, 362, 362, 348, 368, 364]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438612963Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[0]R generation[0..12]: [181, 188, 188, 184, 181, 186, 188, 184, 184, 178, 188, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438631415Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438648357Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[1]R flush_numbers[0..12]: [353, 367, 368, 359, 353, 365, 368, 362, 362, 348, 368, 364]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438664361Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[1]R generation[0..12]: [181, 188, 188, 184, 181, 186, 188, 184, 184, 178, 188, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438679854Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438696487Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[2]R flush_numbers[0..12]: [353, 367, 368, 359, 353, 365, 368, 362, 362, 348, 368, 364]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.43871222Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[2]R generation[0..12]: [181, 188, 188, 184, 181, 186, 188, 184, 184, 178, 188, 186]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438727783Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438744286Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Max found gen is 190","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:55:50.43875798Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Generation requested: 190 >= found:190","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438777511Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Next flush: 370","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438791166Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438807389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"":"downstairs","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438821752Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438834507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"0","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438848381Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"1","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438865613Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"client":"2","":"downstairs","session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438880247Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438893651Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"f1234a74-5966-48e2-903a-d50382b52b2f is now active with session: 92e54e99-5549-4a0a-a573-9e571f167a1a","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438908555Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438921389Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960,"session_id":"92e54e99-5549-4a0a-a573-9e571f167a1a"} {"msg":"The guest has finished waiting for activation with:190","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438934573Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} {"msg":"Activated sub_volume f1234a74-5966-48e2-903a-d50382b52b2f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:50.438946068Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 399 len 8192 data: 9 10 02/30 Read block 28 len 4096 03/30 Write block 419 len 16384 data: 16 15 19 19 04/30 Write block 490 len 20480 data: 18 17 16 14 17 05/30 Write block 38 len 4096 data: 12 06/30 Write block 337 len 4096 data: 17 07/30 Read block 566 len 40960 08/30 Read block 400 len 16384 09/30 Write block 184 len 32768 data: 14 14 16 14 13 13 14 16 10/30 Read block 16 len 28672 11/30 Read block 218 len 40960 12/30 Read block 179 len 16384 13/30 Read block 287 len 12288 14/30 Flush 15/30 Write block 506 len 28672 data: 19 19 19 18 19 19 17 16/30 Write block 460 len 28672 data: 10 12 11 14 16 14 13 17/30 Read block 146 len 32768 18/30 Flush 19/30 Write block 520 len 4096 data: 16 20/30 Write block 214 len 36864 data: 10 10 8 8 9 11 9 12 13 21/30 Read block 418 len 36864 22/30 Write block 567 len 40960 data: 15 17 21 18 15 21 20 18 16 14 23/30 Write block 360 len 16384 data: 18 19 15 16 24/30 Read block 138 len 28672 25/30 Write block 467 len 40960 data: 15 18 17 16 15 14 13 13 12 7 26/30 Read block 179 len 12288 27/30 Write block 456 len 12288 data: 16 14 9 28/30 Write block 162 len 20480 data: 19 15 12 9 10 29/30 Read block 66 len 24576 30/30 Write block 466 len 36864 data: 14 16 19 18 17 16 15 14 14 ---------------------------------------------------------------- Crucible gen:190 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 2 Done Done Done false 1001 Acked Read 1 Done Done Done false 1002 Acked Write 4 Done Done Done false 1003 Acked Write 5 Done Done Done false 1004 Acked Write 1 Done Done Done false 1005 Acked Write 1 Done Done Done false 1006 Acked Read 10 Done Done Done false 1007 Acked Read 4 Done Done Done false 1008 Acked Write 8 Done Done Done false 1009 Acked Read 7 Done Done Done false 1010 Acked Read 10 Done Done Done false 1011 Acked Read 4 Done Done Done false 1012 Acked Read 3 Done Done Done false 1013 Acked Flush 0 Done Done Sent false 1014 Acked Write 7 Done Done Sent false 1015 Acked Write 7 Done Done Sent false 1016 Acked Read 8 Done Done Sent false 1017 Acked Flush 0 Done Done Sent false 1018 Acked Write 1 Done Done Sent false 1019 Acked Write 9 Done Done Sent false 1020 Acked Read 9 Done Done Sent false 1021 Acked Write 10 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Read 7 Done Done Sent false 1024 Acked Write 10 Done Done Sent false 1025 Acked Read 3 Done Done Sent false 1026 Acked Write 3 Done Done Sent false 1027 Acked Write 5 Done Done Sent false 1028 Acked Read 6 Done Done Sent false 1029 Acked Write 9 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 1 17 19 Done 29 29 13 71 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1017)) Some(JobId(1017)) None Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:50.451357539Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1960} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 190 190 188 370 370 367 F F T <--- 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 186 186 186 365 365 365 F F F 6 120-139 188 188 188 368 368 368 F F F 7 140-159 184 184 184 362 362 362 F F F 8 160-179 184 184 184 362 362 362 T T F <--- 9 180-199 190 190 178 370 370 348 F F T <--- 10 200-219 188 188 188 368 368 368 T T F <--- 11 220-239 186 186 186 364 364 364 T T F <--- 12 240-259 189 189 189 369 369 369 F F F 13 260-279 183 183 183 358 358 358 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 190 190 182 370 370 357 F F T <--- 17 340-359 188 188 188 368 368 368 F F F 18 360-379 185 185 185 363 363 363 T T F <--- 19 380-399 190 190 187 370 370 366 F F T <--- 20 400-419 190 190 189 370 370 369 F F T <--- 21 420-439 190 190 189 370 370 369 F F T <--- 22 440-459 188 188 188 368 368 368 T T F <--- 23 460-479 190 190 180 371 371 350 T T F <--- 24 480-499 190 190 188 370 370 368 F F T <--- 25 500-519 190 190 188 371 371 368 F F F <--- 26 520-539 186 186 186 364 364 364 T T F <--- 27 540-559 186 186 186 364 364 364 F F F 28 560-579 189 189 189 369 369 369 T T F <--- 29 580-599 187 187 187 366 366 366 F F F Max gen: 190, Max flush: 371 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 95 Verifying data now No change after verify Loop: 95 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 F F F 1 020-039 190 190 190 370 370 370 F F F 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 186 186 186 365 365 365 F F F 6 120-139 188 188 188 368 368 368 F F F 7 140-159 184 184 184 362 362 362 F F F 8 160-179 191 191 191 372 372 372 F F F 9 180-199 190 190 190 370 370 370 F F F 10 200-219 191 191 191 372 372 372 F F F 11 220-239 191 191 191 372 372 372 F F F 12 240-259 189 189 189 369 369 369 F F F 13 260-279 183 183 183 358 358 358 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 190 190 190 370 370 370 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 191 191 191 372 372 372 F F F 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 190 190 190 370 370 370 F F F 22 440-459 191 191 191 372 372 372 F F F 23 460-479 191 191 191 372 372 372 F F F 24 480-499 190 190 190 370 370 370 F F F 25 500-519 190 190 190 371 371 371 F F F 26 520-539 191 191 191 372 372 372 F F F 27 540-559 186 186 186 364 364 364 F F F 28 560-579 191 191 191 372 372 372 F F F 29 580-599 187 187 187 366 366 366 F F F Max gen: 191, Max flush: 372 Begin loop 96 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.790658549Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.802426019Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.802751588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.802794739Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.802807534Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.802829105Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} {"msg":"Crucible 19817a13-4ba0-4b5e-86c4-c7977d412464 has session id: 55418dfb-8373-4e8c-8de4-6054c0133c9c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803147457Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Upstairs opts: Upstairs UUID: 19817a13-4ba0-4b5e-86c4-c7977d412464, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803181932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Crucible stats registered with UUID: 19817a13-4ba0-4b5e-86c4-c7977d412464","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803197395Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:54.803210699Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803382145Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803411992Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803426786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803580199Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"task":"crutest"} {"msg":"The guest has requested activation with gen:192","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803604219Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} {"msg":"19817a13-4ba0-4b5e-86c4-c7977d412464 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803620602Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803633946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"0","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803648519Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"1","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803662833Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"2","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803676937Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803726826Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803747607Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:62758)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.80376307Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"2","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:65322)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803911605Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"0","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"ds_connection connected from Ok(127.0.0.1:63727)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.803936585Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"io task","client":"1","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.804483008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"2","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.804573359Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"0","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.804759768Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"1","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.804838704Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.804897178Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[0]R flush_numbers[0..12]: [353, 370, 368, 359, 353, 365, 368, 362, 372, 370, 372, 372]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805112365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[0]R generation[0..12]: [181, 190, 188, 184, 181, 186, 188, 184, 191, 190, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805132067Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805148669Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[1]R flush_numbers[0..12]: [353, 370, 368, 359, 353, 365, 368, 362, 372, 370, 372, 372]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805164792Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[1]R generation[0..12]: [181, 190, 188, 184, 181, 186, 188, 184, 191, 190, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805184044Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805200647Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[2]R flush_numbers[0..12]: [353, 370, 368, 359, 353, 365, 368, 362, 372, 370, 372, 372]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.80521641Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[2]R generation[0..12]: [181, 190, 188, 184, 181, 186, 188, 184, 191, 190, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805232603Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805249216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} Wait for a query_work_queue command to finish before sending IO {"msg":"Max found gen is 192","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805268168Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Generation requested: 192 >= found:192","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805282501Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Next flush: 373","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805295786Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.80530932Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"":"downstairs","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805323344Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805336058Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"0","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 2026-03-12T22:55:54.805350522Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"1","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805366975Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"client":"2","":"downstairs","session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805381588Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805395023Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"19817a13-4ba0-4b5e-86c4-c7977d412464 is now active with session: 55418dfb-8373-4e8c-8de4-6054c0133c9c","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805411965Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.80542461Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968,"session_id":"55418dfb-8373-4e8c-8de4-6054c0133c9c"} {"msg":"The guest has finished waiting for activation with:192","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805438204Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} {"msg":"Activated sub_volume 19817a13-4ba0-4b5e-86c4-c7977d412464","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:54.805449729Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 546 len 20480 data: 12 15 17 15 16 02/30 Write block 120 len 40960 data: 11 14 18 18 19 21 19 17 13 13 03/30 Read block 161 len 8192 04/30 Write block 261 len 8192 data: 17 17 05/30 Flush 06/30 Write block 534 len 28672 data: 13 12 12 9 8 11 12 07/30 Write block 110 len 20480 data: 15 14 12 12 9 08/30 Read block 288 len 36864 09/30 Read block 85 len 4096 10/30 Read block 439 len 20480 11/30 Write block 189 len 12288 data: 14 15 17 12/30 Write block 518 len 12288 data: 16 16 17 13/30 Read block 565 len 36864 14/30 Flush 15/30 Write block 365 len 4096 data: 10 16/30 Flush 17/30 Write block 554 len 28672 data: 8 6 11 12 13 15 15 18/30 Read block 477 len 20480 19/30 Write block 154 len 20480 data: 15 17 16 14 16 20/30 Write block 427 len 24576 data: 12 12 18 16 14 10 21/30 Read block 69 len 40960 22/30 Read block 571 len 36864 23/30 Read block 423 len 36864 24/30 Write block 13 len 12288 data: 15 18 16 25/30 Write block 552 len 16384 data: 9 8 9 7 26/30 Read block 58 len 40960 27/30 Write block 262 len 12288 data: 18 13 15 28/30 Write block 446 len 20480 data: 10 9 10 9 12 29/30 Read block 525 len 28672 30/30 Read block 61 len 36864 ---------------------------------------------------------------- Crucible gen:192 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 10 Done Done Done false 1002 Acked Read 2 Done Sent Done false 1003 Acked Write 2 Done Sent Done false 1004 Acked Flush 0 Done Sent Done false 1005 Acked Write 7 Done Sent Done false 1006 Acked Write 5 Done Sent Done false 1007 Acked Read 9 Done Sent Done false 1008 Acked Read 1 Done Sent Done false 1009 Acked Read 5 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 3 Done Sent Done false 1012 Acked Read 9 Done Sent Done false 1013 Acked Flush 0 Done Sent Done false 1014 Acked Write 1 Done Sent Done false 1015 Acked Flush 0 Done Sent Done false 1016 Acked Write 7 Done Sent Done false 1017 Acked Read 5 Done Sent Done false 1018 Acked Write 5 Done Sent Done false 1019 Acked Write 6 Done Sent Done false 1020 Acked Read 10 Done Sent Done false 1021 Acked Read 9 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 3 Done Sent Done false 1024 Acked Write 4 Done Sent Done false 1025 Acked Read 10 Done Sent Done false 1026 Acked Write 3 Done Sent Done false 1027 Acked Write 5 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Read 9 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 28 1 29 Done 30 2 29 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1015)) None Some(JobId(1015)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:54.816771742Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1968} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 181 181 181 353 353 353 T F T <--- 1 020-039 190 190 190 370 370 370 F F F 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 192 186 192 374 365 374 F F F <--- 6 120-139 192 188 192 373 368 373 F T F <--- 7 140-159 184 184 184 362 362 362 T F T <--- 8 160-179 191 191 191 372 372 372 F F F 9 180-199 192 190 192 374 370 374 F F F <--- 10 200-219 191 191 191 372 372 372 F F F 11 220-239 191 191 191 372 372 372 F F F 12 240-259 189 189 189 369 369 369 F F F 13 260-279 192 183 192 373 358 373 T F T <--- 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 190 190 190 370 370 370 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 192 191 192 375 372 375 F F F <--- 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 190 190 190 370 370 370 T F T <--- 22 440-459 191 191 191 372 372 372 T F T <--- 23 460-479 191 191 191 372 372 372 F F F 24 480-499 190 190 190 370 370 370 F F F 25 500-519 192 190 192 374 371 374 F F F <--- 26 520-539 192 191 192 374 372 374 F F F <--- 27 540-559 192 186 192 374 364 374 T T T <--- 28 560-579 191 191 191 372 372 372 T F T <--- 29 580-599 187 187 187 366 366 366 F F F Max gen: 192, Max flush: 375 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 96 Verifying data now No change after verify Loop: 96 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 376 376 376 F F F 1 020-039 190 190 190 370 370 370 F F F 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 181 181 181 353 353 353 F F F 5 100-119 192 192 192 374 374 374 F F F 6 120-139 192 192 192 373 373 373 F F F 7 140-159 193 193 193 376 376 376 F F F 8 160-179 191 191 191 372 372 372 F F F 9 180-199 192 192 192 374 374 374 F F F 10 200-219 191 191 191 372 372 372 F F F 11 220-239 191 191 191 372 372 372 F F F 12 240-259 189 189 189 369 369 369 F F F 13 260-279 193 193 193 376 376 376 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 190 190 190 370 370 370 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 192 192 192 375 375 375 F F F 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 193 193 193 376 376 376 F F F 22 440-459 193 193 193 376 376 376 F F F 23 460-479 191 191 191 372 372 372 F F F 24 480-499 190 190 190 370 370 370 F F F 25 500-519 192 192 192 374 374 374 F F F 26 520-539 192 192 192 374 374 374 F F F 27 540-559 193 193 193 376 376 376 F F F 28 560-579 193 193 193 376 376 376 F F F 29 580-599 187 187 187 366 366 366 F F F Max gen: 193, Max flush: 376 Begin loop 97 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.151943271Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.163671668Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.163984323Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164023656Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.16403604Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164054162Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} {"msg":"Crucible 7a680263-077f-4c72-867a-b478f0810a8f has session id: 0a9d6e3b-ce07-4a26-8dcc-c229687e1e10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164410098Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Upstairs opts: Upstairs UUID: 7a680263-077f-4c72-867a-b478f0810a8f, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164445583Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Crucible stats registered with UUID: 7a680263-077f-4c72-867a-b478f0810a8f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164461676Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:59.16447524Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164662059Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164701822Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164717745Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164840052Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"task":"crutest"} {"msg":"The guest has requested activation with gen:194","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164857584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} {"msg":"7a680263-077f-4c72-867a-b478f0810a8f active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164872658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164885882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"0","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164900216Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"1","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.16491455Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"2","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164929043Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164968226Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.164991276Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"ds_connection connected from Ok(127.0.0.1:44831)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.165007249Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"2","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"ds_connection connected from Ok(127.0.0.1:52024)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.165022593Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"0","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"ds_connection connected from Ok(127.0.0.1:52412)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.165037736Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"io task","client":"1","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.165725448Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"0","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.165839859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"2","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.165875913Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"1","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166108472Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166160689Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[0]R flush_numbers[0..12]: [376, 370, 368, 359, 353, 374, 373, 376, 372, 374, 372, 372]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166267493Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[0]R generation[0..12]: [193, 190, 188, 184, 181, 192, 192, 193, 191, 192, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166285885Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166303078Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[1]R flush_numbers[0..12]: [376, 370, 368, 359, 353, 374, 373, 376, 372, 374, 372, 372]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166319321Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[1]R generation[0..12]: [193, 190, 188, 184, 181, 192, 192, 193, 191, 192, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166335394Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166352167Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[2]R flush_numbers[0..12]: [376, 370, 368, 359, 353, 374, 373, 376, 372, 374, 372, 372]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166368429Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[2]R generation[0..12]: [193, 190, 188, 184, 181, 192, 192, 193, 191, 192, 191, 191]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166384942Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166402055Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Max found gen is 194","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:55:59.166416429Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Generation requested: 194 >= found:194","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166453553Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Next flush: 377","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166467766Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.16648172Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"":"downstairs","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166496574Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166509848Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"0","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166524402Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"1","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166538966Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"client":"2","":"downstairs","session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166557058Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166571142Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"7a680263-077f-4c72-867a-b478f0810a8f is now active with session: 0a9d6e3b-ce07-4a26-8dcc-c229687e1e10","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166586945Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166600379Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976,"session_id":"0a9d6e3b-ce07-4a26-8dcc-c229687e1e10"} {"msg":"The guest has finished waiting for activation with:194","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166614313Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} {"msg":"Activated sub_volume 7a680263-077f-4c72-867a-b478f0810a8f","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:55:59.166626278Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Flush 02/30 Write block 537 len 40960 data: 10 9 12 13 12 12 12 12 15 13 03/30 Write block 560 len 8192 data: 16 15 04/30 Write block 83 len 12288 data: 16 16 15 05/30 Write block 432 len 4096 data: 11 06/30 Read block 161 len 36864 07/30 Read block 424 len 8192 08/30 Write block 478 len 12288 data: 13 15 18 09/30 Read block 206 len 36864 10/30 Write block 159 len 16384 data: 16 20 19 20 11/30 Read block 356 len 36864 12/30 Read block 42 len 16384 13/30 Flush 14/30 Write block 422 len 4096 data: 20 15/30 Flush 16/30 Write block 523 len 20480 data: 10 10 16 13 14 17/30 Read block 187 len 28672 18/30 Read block 131 len 24576 19/30 Read block 436 len 28672 20/30 Read block 207 len 28672 21/30 Write block 144 len 20480 data: 14 15 14 17 17 22/30 Read block 178 len 32768 23/30 Read block 578 len 36864 24/30 Write block 526 len 24576 data: 14 15 15 17 15 17 25/30 Flush 26/30 Write block 205 len 36864 data: 22 15 14 13 14 11 11 12 10 27/30 Write block 231 len 16384 data: 10 13 15 17 28/30 Write block 243 len 28672 data: 15 14 14 15 13 14 14 29/30 Read block 537 len 28672 30/30 Write block 563 len 28672 data: 16 14 15 13 16 18 22 ---------------------------------------------------------------- Crucible gen:194 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Flush 0 Done Sent Done false 1001 Acked Write 10 Done Sent Done false 1002 Acked Write 2 Done Sent Done false 1003 Acked Write 3 Done Sent Done false 1004 Acked Write 1 Done Sent Done false 1005 Acked Read 9 Done Sent Done false 1006 Acked Read 2 Done Sent Done false 1007 Acked Write 3 Done Sent Done false 1008 Acked Read 9 Done Sent Done false 1009 Acked Write 4 Done Sent Done false 1010 Acked Read 9 Done Sent Done false 1011 Acked Read 4 Done Sent Done false 1012 Acked Flush 0 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Flush 0 Done Sent Done false 1015 Acked Write 5 Done Sent Done false 1016 Acked Read 7 Done Sent Done false 1017 Acked Read 6 Done Sent Done false 1018 Acked Read 7 Done Sent Done false 1019 Acked Read 7 Done Sent Done false 1020 Acked Write 5 Done Sent Done false 1021 Acked Read 8 Done Sent Done false 1022 Acked Read 9 Done Sent Done false 1023 Acked Write 6 Done Sent Done false 1024 Acked Flush 0 Done Sent Done false 1025 Acked Write 9 Done Sent Done false 1026 Acked Write 4 Done Sent Done false 1027 Acked Write 7 Done Sent Done false 1028 Acked Read 7 Done Sent Done false 1029 Acked Write 7 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 1 30 1 32 Done 29 0 29 58 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1024)) None Some(JobId(1024)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:55:59.178482859Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1976} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 376 376 376 F F F 1 020-039 190 190 190 370 370 370 F F F 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 194 181 194 378 353 378 F F F <--- 5 100-119 192 192 192 374 374 374 F F F 6 120-139 192 192 192 373 373 373 F F F 7 140-159 194 193 194 380 376 380 F F F <--- 8 160-179 194 191 194 378 372 378 F F F <--- 9 180-199 192 192 192 374 374 374 F F F 10 200-219 191 191 191 372 372 372 T F T <--- 11 220-239 191 191 191 372 372 372 T F T <--- 12 240-259 189 189 189 369 369 369 T F T <--- 13 260-279 193 193 193 376 376 376 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 190 190 190 370 370 370 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 192 192 192 375 375 375 F F F 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 194 193 194 379 376 379 F F F <--- 22 440-459 193 193 193 376 376 376 F F F 23 460-479 194 191 194 378 372 378 F F F <--- 24 480-499 194 190 194 378 370 378 F F F <--- 25 500-519 192 192 192 374 374 374 F F F 26 520-539 194 192 194 380 374 380 F F F <--- 27 540-559 194 193 194 378 376 378 F F F <--- 28 560-579 194 193 194 378 376 378 T F T <--- 29 580-599 187 187 187 366 366 366 F F F Max gen: 194, Max flush: 380 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 97 Verifying data now No change after verify Loop: 97 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 193 193 193 376 376 376 F F F 1 020-039 190 190 190 370 370 370 F F F 2 040-059 188 188 188 368 368 368 F F F 3 060-079 184 184 184 359 359 359 F F F 4 080-099 194 194 194 378 378 378 F F F 5 100-119 192 192 192 374 374 374 F F F 6 120-139 192 192 192 373 373 373 F F F 7 140-159 194 194 194 380 380 380 F F F 8 160-179 194 194 194 378 378 378 F F F 9 180-199 192 192 192 374 374 374 F F F 10 200-219 195 195 195 381 381 381 F F F 11 220-239 195 195 195 381 381 381 F F F 12 240-259 195 195 195 381 381 381 F F F 13 260-279 193 193 193 376 376 376 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 188 188 188 368 368 368 F F F 16 320-339 190 190 190 370 370 370 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 192 192 192 375 375 375 F F F 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 194 194 194 379 379 379 F F F 22 440-459 193 193 193 376 376 376 F F F 23 460-479 194 194 194 378 378 378 F F F 24 480-499 194 194 194 378 378 378 F F F 25 500-519 192 192 192 374 374 374 F F F 26 520-539 194 194 194 380 380 380 F F F 27 540-559 194 194 194 378 378 378 F F F 28 560-579 195 195 195 381 381 381 F F F 29 580-599 187 187 187 366 366 366 F F F Max gen: 195, Max flush: 381 Begin loop 98 Downstairs to restart: 2 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.516493118Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.528271563Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.528612925Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.528652738Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.528668051Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.528686234Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} {"msg":"Crucible 2948c2cb-4956-49cb-a5af-fc53b0deda26 has session id: 40d04ba6-b4a3-409f-ae20-69f972913458","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529052215Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Upstairs opts: Upstairs UUID: 2948c2cb-4956-49cb-a5af-fc53b0deda26, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.52908743Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Crucible stats registered with UUID: 2948c2cb-4956-49cb-a5af-fc53b0deda26","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529103173Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:56:03.529116637Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529306994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529342009Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529357643Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529530837Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"task":"crutest"} {"msg":"The guest has requested activation with gen:196","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529555507Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} {"msg":"2948c2cb-4956-49cb-a5af-fc53b0deda26 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529568131Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529581316Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"0","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529595869Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"1","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529610243Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"2","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529631974Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529652455Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529666679Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"ds_connection connected from Ok(127.0.0.1:35547)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529681882Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"2","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"ds_connection connected from Ok(127.0.0.1:61908)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529696806Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"0","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"ds_connection connected from Ok(127.0.0.1:47487)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.529812445Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"io task","client":"1","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530428039Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"0","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530453478Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"1","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.53047008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"2","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530806035Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530822658Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[0]R flush_numbers[0..12]: [376, 370, 368, 359, 378, 374, 373, 380, 378, 374, 381, 381]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530856863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[0]R generation[0..12]: [193, 190, 188, 184, 194, 192, 192, 194, 194, 192, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530874325Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530890888Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[1]R flush_numbers[0..12]: [376, 370, 368, 359, 378, 374, 373, 380, 378, 374, 381, 381]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.53090988Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[1]R generation[0..12]: [193, 190, 188, 184, 194, 192, 192, 194, 194, 192, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530926772Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530945464Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[2]R flush_numbers[0..12]: [376, 370, 368, 359, 378, 374, 373, 380, 378, 374, 381, 381]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530961717Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[2]R generation[0..12]: [193, 190, 188, 184, 194, 192, 192, 194, 194, 192, 195, 195]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.53097762Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.530995013Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Max found gen is 196","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531008767Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Generation requested: 196 >= found:196","v":0,"name":"crucible","level":30Wait for a query_work_queue command to finish before sending IO ,"time":"2026-03-12T22:56:03.531023221Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Next flush: 382","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531042552Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531057066Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"":"downstairs","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.53107148Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531084194Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"0","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531097938Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"1","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531112702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"client":"2","":"downstairs","session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 ,"time":"2026-03-12T22:56:03.531126826Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531145028Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"2948c2cb-4956-49cb-a5af-fc53b0deda26 is now active with session: 40d04ba6-b4a3-409f-ae20-69f972913458","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531159971Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531172556Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984,"session_id":"40d04ba6-b4a3-409f-ae20-69f972913458"} {"msg":"The guest has finished waiting for activation with:196","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.53118613Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} {"msg":"Activated sub_volume 2948c2cb-4956-49cb-a5af-fc53b0deda26","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:03.531198015Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 12 len 40960 data: 14 16 19 17 19 19 17 19 20 22 02/30 Read block 207 len 12288 03/30 Flush 04/30 Write block 580 len 8192 data: 12 12 05/30 Read block 422 len 24576 06/30 Read block 229 len 24576 07/30 Write block 226 len 4096 data: 8 08/30 Write block 305 len 16384 data: 11 12 12 12 09/30 Flush 10/30 Write block 544 len 24576 data: 13 16 14 16 18 16 11/30 Write block 506 len 36864 data: 20 20 20 19 20 20 18 13 13 12/30 Read block 23 len 40960 13/30 Read block 425 len 12288 14/30 Read block 433 len 8192 15/30 Read block 557 len 8192 16/30 Write block 326 len 16384 data: 15 16 19 15 17/30 Write block 225 len 36864 data: 9 9 9 9 10 10 11 14 16 18/30 Write block 254 len 16384 data: 15 16 15 18 19/30 Write block 148 len 36864 data: 18 17 16 16 12 11 16 18 17 20/30 Read block 444 len 24576 21/30 Write block 46 len 8192 data: 13 13 22/30 Read block 394 len 32768 23/30 Write block 68 len 16384 data: 10 9 9 8 24/30 Write block 172 len 36864 data: 18 13 14 15 13 12 8 11 12 25/30 Read block 337 len 8192 26/30 Read block 388 len 36864 27/30 Write block 585 len 24576 data: 13 15 14 15 17 20 28/30 Read block 86 len 8192 29/30 Read block 514 len 8192 30/30 Read block 527 len 24576 ---------------------------------------------------------------- Crucible gen:196 GIO:true work queues: Upstairs:0 downstairs:27 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1003 Acked Write 2 Done Done Done false 1004 Acked Read 6 Done Done Sent false 1005 Acked Read 6 Done Done Sent false 1006 Acked Write 1 Done Done Sent false 1007 Acked Write 4 Done Done Sent false 1008 Acked Flush 0 Done Done Sent false 1009 Acked Write 6 Done Done Sent false 1010 Acked Write 9 Done Done Sent false 1011 Acked Read 10 Done Done Sent false 1012 Acked Read 3 Done Done Sent false 1013 Acked Read 2 Done Done Sent false 1014 Acked Read 2 Done Done Sent false 1015 Acked Write 4 Done Done Sent false 1016 Acked Write 9 Done Done Sent false 1017 Acked Write 4 Done Done Sent false 1018 Acked Write 9 Done Done Sent false 1019 Acked Read 6 Done Done Sent false 1020 Acked Write 2 Done Done Sent false 1021 Acked Read 8 Done Done Sent false 1022 Acked Write 4 Done Done Sent false 1023 Acked Write 9 Done Done Sent false 1024 Acked Read 2 Done Done Sent false 1025 Acked Read 9 Done Done Sent false 1026 Acked Write 6 Done Done Sent false 1027 Acked Read 2 Done Done Sent false 1028 Acked Read 2 Done Done Sent false 1029 Acked Read 6 Done Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 0 1 26 27 Done 27 26 1 54 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1008)) Some(JobId(1008)) Some(JobId(1002)) Downstairs last five completed: 1002 1001 1000 Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:56:03.543147026Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1984} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 382 382 382 F F F 1 020-039 196 196 196 382 382 382 F F F 2 040-059 188 188 188 368 368 368 T T F <--- 3 060-079 184 184 184 359 359 359 T T F <--- 4 080-099 194 194 194 378 378 378 F F F 5 100-119 192 192 192 374 374 374 F F F 6 120-139 192 192 192 373 373 373 F F F 7 140-159 194 194 194 380 380 380 T T F <--- 8 160-179 194 194 194 378 378 378 T T F <--- 9 180-199 192 192 192 374 374 374 T T F <--- 10 200-219 195 195 195 381 381 381 F F F 11 220-239 196 196 195 383 383 381 T T F <--- 12 240-259 195 195 195 381 381 381 T T F <--- 13 260-279 193 193 193 376 376 376 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 196 196 188 383 383 368 F F F <--- 16 320-339 190 190 190 370 370 370 T T F <--- 17 340-359 188 188 188 368 368 368 F F F 18 360-379 192 192 192 375 375 375 F F F 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 194 194 194 379 379 379 F F F 22 440-459 193 193 193 376 376 376 F F F 23 460-479 194 194 194 378 378 378 F F F 24 480-499 194 194 194 378 378 378 F F F 25 500-519 192 192 192 374 374 374 T T F <--- 26 520-539 194 194 194 380 380 380 F F F 27 540-559 194 194 194 378 378 378 T T F <--- 28 560-579 195 195 195 381 381 381 F F F 29 580-599 196 196 187 383 383 366 T T T <--- Max gen: 196, Max flush: 383 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 98 Verifying data now No change after verify Loop: 98 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 196 196 196 382 382 382 F F F 1 020-039 196 196 196 382 382 382 F F F 2 040-059 197 197 197 384 384 384 F F F 3 060-079 197 197 197 384 384 384 F F F 4 080-099 194 194 194 378 378 378 F F F 5 100-119 192 192 192 374 374 374 F F F 6 120-139 192 192 192 373 373 373 F F F 7 140-159 197 197 197 384 384 384 F F F 8 160-179 197 197 197 384 384 384 F F F 9 180-199 197 197 197 384 384 384 F F F 10 200-219 195 195 195 381 381 381 F F F 11 220-239 197 197 197 384 384 384 F F F 12 240-259 197 197 197 384 384 384 F F F 13 260-279 193 193 193 376 376 376 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 196 196 196 383 383 383 F F F 16 320-339 197 197 197 384 384 384 F F F 17 340-359 188 188 188 368 368 368 F F F 18 360-379 192 192 192 375 375 375 F F F 19 380-399 190 190 190 370 370 370 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 194 194 194 379 379 379 F F F 22 440-459 193 193 193 376 376 376 F F F 23 460-479 194 194 194 378 378 378 F F F 24 480-499 194 194 194 378 378 378 F F F 25 500-519 197 197 197 384 384 384 F F F 26 520-539 194 194 194 380 380 380 F F F 27 540-559 197 197 197 384 384 384 F F F 28 560-579 195 195 195 381 381 381 F F F 29 580-599 197 197 197 384 384 384 F F F Max gen: 197, Max flush: 384 Begin loop 99 Downstairs to restart: 1 {"msg":"look at: http://127.0.0.1:12810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.872179046Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"task":"crutest"} {"msg":"RI is: RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 12345678-0000-0000-0000-000000008810, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.883844431Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"task":"crutest"} {"msg":"Upstairs starts","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884153597Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} {"msg":"Crucible Version: BuildInfo {\n version: \"0.0.1\",\n git_sha: \"89e6341ecf66caa93ce65bef5941ee35ced6b768\",\n git_commit_timestamp: \"2026-03-12T22:28:21.000000000Z\",\n git_branch: \"alan/rinque\",\n rustc_semver: \"1.90.0\",\n rustc_channel: \"stable\",\n rustc_host_triple: \"x86_64-unknown-illumos\",\n rustc_commit_sha: \"1159e78c4747b02ef996e55082b704c09b970588\",\n cargo_triple: \"x86_64-unknown-illumos\",\n debug: true,\n opt_level: 0,\n}","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.88419113Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} {"msg":"Upstairs <-> Downstairs Message Version: 13","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884203775Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} {"msg":"Using region definition RegionDefinition { block_size: 4096, extent_size: Block { value: 20, shift: 12 }, extent_count: 30, uuid: 00000000-0000-0000-0000-000000000000, encrypted: false, database_read_version: 1, database_write_version: 1 }","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884221727Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} {"msg":"Crucible 6ac2e38a-2769-4687-ac6e-277927fbf945 has session id: daa94783-3508-4945-bc83-caa1a9bb2972","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884574444Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Upstairs opts: Upstairs UUID: 6ac2e38a-2769-4687-ac6e-277927fbf945, Targets: [127.0.0.1:8810, 127.0.0.1:8820, 127.0.0.1:8830], lossy: false, flush_timeout: None, key populated: false, cert_pem populated: false, key_pem populated: false, root_cert_pem populated: false, Control: None, read_only: false","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.88460838Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Crucible stats registered with UUID: 6ac2e38a-2769-4687-ac6e-277927fbf945","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884623863Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"could not find Downstairs address for Nexus","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:56:07.884637417Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884803365Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"2","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884829354Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"1","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"client is waiting for oneshot","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884843897Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"0","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Downstairs in volume = 3","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884972702Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"task":"crutest"} {"msg":"The guest has requested activation with gen:198","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.884990284Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} {"msg":"6ac2e38a-2769-4687-ac6e-277927fbf945 active request set","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.88502349Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885037454Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"0","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885051547Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"1","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"sending connect oneshot to client","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885065192Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"2","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"connecting to 127.0.0.1:8830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885078946Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"2","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"connecting to 127.0.0.1:8810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885093139Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"0","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"connecting to 127.0.0.1:8820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885107283Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"1","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"ds_connection connected from Ok(127.0.0.1:51621)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885212008Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"2","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"ds_connection connected from Ok(127.0.0.1:52922)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885251461Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"0","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"ds_connection connected from Ok(127.0.0.1:35251)","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885268044Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"io task","client":"1","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"downstairs client at Some(127.0.0.1:8830) has region UUID 12345678-0000-0000-0000-000000008830","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885917982Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"2","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"downstairs client at Some(127.0.0.1:8810) has region UUID 12345678-0000-0000-0000-000000008810","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.885957025Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"0","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"downstairs client at Some(127.0.0.1:8820) has region UUID 12345678-0000-0000-0000-000000008820","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886072495Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"1","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886309822Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Waiting for more clients to be ready","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886324306Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[0]R flush_numbers[0..12]: [382, 382, 384, 384, 378, 374, 373, 384, 384, 384, 381, 384]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886471782Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[0]R generation[0..12]: [196, 196, 197, 197, 194, 192, 192, 197, 197, 197, 195, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886488994Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[0]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886505517Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[1]R flush_numbers[0..12]: [382, 382, 384, 384, 378, 374, 373, 384, 384, 384, 381, 384]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886521201Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[1]R generation[0..12]: [196, 196, 197, 197, 194, 192, 192, 197, 197, 197, 195, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886536894Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[1]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886552967Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[2]R flush_numbers[0..12]: [382, 382, 384, 384, 378, 374, 373, 384, 384, 384, 381, 384]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.88656836Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[2]R generation[0..12]: [196, 196, 197, 197, 194, 192, 192, 197, 197, 197, 195, 197]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886584153Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"[2]R dirty[0..12]: [false, false, false, false, false, false, false, false, false, false, false, false]","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886606544Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Max found gen is 198","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886620368Z","hostname":Wait for a query_work_queue command to finish before sending IO "w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Generation requested: 198 >= found:198","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886643308Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Next flush: 385","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886656672Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"All extents match","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886670196Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"":"downstairs","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"No downstairs reconciliation required","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.88668418Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886696744Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"0","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886710528Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"1","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Disk: sv:1 bs:4096 ts:2457600 tb:600 max_io:256 or 1048576 Transition from WaitQuorum to Active","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886726761Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"client":"2","":"downstairs","session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"All required reconciliation work is completed","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886741415Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Set Downstairs and Upstairs active after reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886754959Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"6ac2e38a-2769-4687-ac6e-277927fbf945 is now active with session: daa94783-3508-4945-bc83-caa1a9bb2972","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886769633Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"Set Active after no reconciliation","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886782407Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992,"session_id":"daa94783-3508-4945-bc83-caa1a9bb2972"} {"msg":"The guest has finished waiting for activation with:198","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886796311Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} {"msg":"Activated sub_volume 6ac2e38a-2769-4687-ac6e-277927fbf945","v":0,"name":"crucible","level":30,"time":"2026-03-12T22:56:07.886807956Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} Loading write count information from file "/tmp/test_live_repair/test_repair_verify.data" Run Repair workload 01/30 Write block 380 len 20480 data: 19 19 19 16 14 02/30 Write block 9 len 32768 data: 13 14 14 15 17 20 18 20 03/30 Write block 349 len 20480 data: 17 19 20 19 17 04/30 Write block 475 len 12288 data: 13 8 8 05/30 Write block 210 len 20480 data: 12 12 13 11 11 06/30 Read block 576 len 20480 07/30 Write block 94 len 4096 data: 18 08/30 Read block 410 len 8192 09/30 Write block 263 len 8192 data: 14 16 10/30 Read block 92 len 4096 11/30 Write block 457 len 12288 data: 15 10 11 12/30 Write block 226 len 16384 data: 10 10 10 11 13/30 Read block 17 len 24576 14/30 Write block 546 len 4096 data: 15 15/30 Read block 251 len 12288 16/30 Read block 306 len 8192 17/30 Read block 434 len 32768 18/30 Write block 537 len 36864 data: 11 10 13 14 13 13 13 14 17 19/30 Read block 573 len 20480 20/30 Flush 21/30 Write block 502 len 16384 data: 19 18 17 17 22/30 Read block 572 len 12288 23/30 Read block 160 len 20480 24/30 Read block 418 len 28672 25/30 Write block 50 len 28672 data: 14 14 11 11 13 14 13 26/30 Read block 407 len 36864 27/30 Write block 97 len 24576 data: 16 16 16 20 20 17 28/30 Read block 530 len 16384 29/30 Write block 558 len 28672 data: 14 16 17 16 16 17 15 30/30 Write block 495 len 20480 data: 19 20 19 19 22 ---------------------------------------------------------------- Crucible gen:198 GIO:true work queues: Upstairs:0 downstairs:30 States: Active Active Active JOBID ACK TYPE BKS/EXT DS:0 DS:1 DS:2 REPLAY 1000 Acked Write 5 Done Done Done false 1001 Acked Write 8 Done Done Done false 1002 Acked Write 5 Done Done Done false 1003 Acked Write 3 Done Done Done false 1004 Acked Write 5 Done Done Done false 1005 Acked Read 5 Done Sent Done false 1006 Acked Write 1 Done Sent Done false 1007 Acked Read 2 Done Sent Done false 1008 Acked Write 2 Done Sent Done false 1009 Acked Read 1 Done Sent Done false 1010 Acked Write 3 Done Sent Done false 1011 Acked Write 4 Done Sent Done false 1012 Acked Read 6 Done Sent Done false 1013 Acked Write 1 Done Sent Done false 1014 Acked Read 3 Done Sent Done false 1015 Acked Read 2 Done Sent Done false 1016 Acked Read 8 Done Sent Done false 1017 Acked Write 9 Done Sent Done false 1018 Acked Read 5 Done Sent Done false 1019 Acked Flush 0 Done Sent Done false 1020 Acked Write 4 Done Sent Done false 1021 Acked Read 3 Done Sent Done false 1022 Acked Read 5 Done Sent Done false 1023 Acked Read 7 Done Sent Done false 1024 Acked Write 7 Done Sent Done false 1025 Acked Read 9 Done Sent Done false 1026 Acked Write 6 Done Sent Done false 1027 Acked Read 4 Done Sent Done false 1028 Acked Write 7 Sent Sent Sent false 1029 Acked Write 5 Sent Sent Sent false STATES DS:0 DS:1 DS:2 TOTAL Sent 2 25 2 29 Done 28 5 28 61 Skipped 0 0 0 0 Error 0 0 0 0 Last Flush: Some(JobId(1019)) None Some(JobId(1019)) Downstairs last five completed: Upstairs last five completed: 1029 1028 1027 1026 1025 {"msg":"Guest handle has been dropped","v":0,"name":"crucible","level":40,"time":"2026-03-12T22:56:07.897958584Z","hostname":"w-01KKJ3PP2P40RZY8RK6MQRV8BV","pid":1992} Wrote out file "/tmp/test_live_repair/test_repair_verify.data" Current downstairs dump with dump args: --no-color -d /var/tmp/test_repair/8810 -d /var/tmp/test_repair/8820 -d /var/tmp/test_repair/8830 EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 196 198 385 382 385 F T F <--- 1 020-039 196 196 196 382 382 382 F F F 2 040-059 197 197 197 384 384 384 T F T <--- 3 060-079 197 197 197 384 384 384 F F F 4 080-099 198 194 198 385 378 385 T F T <--- 5 100-119 192 192 192 374 374 374 T F T <--- 6 120-139 192 192 192 373 373 373 F F F 7 140-159 197 197 197 384 384 384 F F F 8 160-179 197 197 197 384 384 384 F F F 9 180-199 197 197 197 384 384 384 F F F 10 200-219 198 195 198 385 381 385 F T F <--- 11 220-239 198 197 198 385 384 385 F F F <--- 12 240-259 197 197 197 384 384 384 F F F 13 260-279 198 193 198 385 376 385 F F F <--- 14 280-299 188 188 188 368 368 368 F F F 15 300-319 196 196 196 383 383 383 F F F 16 320-339 197 197 197 384 384 384 F F F 17 340-359 198 188 198 385 368 385 F T F <--- 18 360-379 192 192 192 375 375 375 F F F 19 380-399 198 190 198 385 370 385 F T F <--- 20 400-419 190 190 190 370 370 370 F F F 21 420-439 194 194 194 379 379 379 F F F 22 440-459 198 193 198 385 376 385 F F F <--- 23 460-479 198 194 198 385 378 385 F T F <--- 24 480-499 194 194 194 378 378 378 T F T <--- 25 500-519 197 197 197 384 384 384 T F T <--- 26 520-539 198 194 198 385 380 385 F F F <--- 27 540-559 198 197 198 385 384 385 T F T <--- 28 560-579 195 195 195 381 381 381 T F T <--- 29 580-599 197 197 197 384 384 384 F F F Max gen: 198, Max flush: 385 Error: Difference in extent metadata found! Stack backtrace: 0: anyhow::error::::msg at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/backtrace.rs:10:14 1: anyhow::__private::format_err at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/anyhow-1.0.102/src/lib.rs:687:13 A Difference in extent metadata is expected here On loop 99 Verifying data now No change after verify Loop: 99 Downstairs dump after verify (and repair): EXT BLOCKS GEN0 GEN1 GEN2 FL0 FL1 FL2 D0 D1 D2 DIFF 0 000-019 198 198 198 385 385 385 F F F 1 020-039 196 196 196 382 382 382 F F F 2 040-059 199 199 199 386 386 386 F F F 3 060-079 197 197 197 384 384 384 F F F 4 080-099 199 199 199 386 386 386 F F F 5 100-119 199 199 199 386 386 386 F F F 6 120-139 192 192 192 373 373 373 F F F 7 140-159 197 197 197 384 384 384 F F F 8 160-179 197 197 197 384 384 384 F F F 9 180-199 197 197 197 384 384 384 F F F 10 200-219 198 198 198 385 385 385 F F F 11 220-239 198 198 198 385 385 385 F F F 12 240-259 197 197 197 384 384 384 F F F 13 260-279 198 198 198 385 385 385 F F F 14 280-299 188 188 188 368 368 368 F F F 15 300-319 196 196 196 383 383 383 F F F 16 320-339 197 197 197 384 384 384 F F F 17 340-359 198 198 198 385 385 385 F F F 18 360-379 192 192 192 375 375 375 F F F 19 380-399 198 198 198 385 385 385 F F F 20 400-419 190 190 190 370 370 370 F F F 21 420-439 194 194 194 379 379 379 F F F 22 440-459 198 198 198 385 385 385 F F F 23 460-479 198 198 198 385 385 385 F F F 24 480-499 199 199 199 386 386 386 F F F 25 500-519 199 199 199 386 386 386 F F F 26 520-539 198 198 198 385 385 385 F F F 27 540-559 199 199 199 386 386 386 F F F 28 560-579 199 199 199 386 386 386 F F F 29 580-599 197 197 197 384 384 384 F F F Max gen: 199, Max flush: 386 7:36 Test duration Test completed real 7:36.111644641 user 35.973699170 sys 17.174090962 trap 0.015079067 tflt 0.083471116 dflt 0.064717231 kflt 0.003401814 lock 11:54:52.882693699 slp 36:38.183192827 lat 1.493640750 stop 0.035309664